diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 000000000..7b26795b0 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,19 @@ +# Change Log + +## Version 10.0.8: + +- Rewrote sync command to eliminate numerous bugs and improve usability (see wiki for details) +- Implemented various improvements to memory management +- Added MD5 validation support (available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing) +- Added last modified time checks for source to guarantee transfer integrity +- Formalized outputs in JSON and elevated the output flag to the root level +- Eliminated outputs to STDERR (for new version notifications), which were causing problems for certain CI systems +- Improved log format for Windows +- Optimized plan file sizes +- Improved command line parameter names as follows (to be consistent with naming pattern of other parameters): + - fromTo -> from-to + - blobType -> blob-type + - excludedBlobType -> excluded-blob-type + - outputRaw (in "list" command) -> output + - stdIn-enable (reserved for internal use) -> stdin-enable + diff --git a/Gopkg.lock b/Gopkg.lock index ad3367d91..6949845dd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -18,12 +18,12 @@ version = "v0.1.8" [[projects]] - digest = "1:b8ac7e4464ce21f7487c663aa69b1b3437740bb10ab12d4dc7aa9b02422571a1" + digest = "1:435043934aa0a8221e2c660e88dffe588783e9497facf6b517a465a37c58c97b" name = "github.com/Azure/azure-storage-blob-go" packages = ["azblob"] pruneopts = "UT" - revision = "45d0c5e3638e2b539942f93c48e419f4f2fc62e4" - version = "0.4.0" + revision = "457680cc0804810f6d02958481e0ffdda51d5c60" + version = "0.5.0" [[projects]] digest = "1:0376b54cf965bdae03b9c2a2734cad87fc3bbaa4066c0374ec86f10f77f03d45" diff --git a/Gopkg.toml b/Gopkg.toml index e9a5f06f7..722aeb665 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -4,7 +4,7 @@ [[constraint]] name = "github.com/Azure/azure-storage-blob-go" - version = "0.4.0" + version = "0.5.0" [[constraint]] name = "github.com/Azure/azure-storage-file-go" diff --git a/azbfs/azure_dfs_swagger.json b/azbfs/azure_dfs_swagger.json index a8bcfc33e..1a8a64dd4 100644 --- a/azbfs/azure_dfs_swagger.json +++ b/azbfs/azure_dfs_swagger.json @@ -1,1760 +1,1885 @@ -{ - "swagger": "2.0", - "info": { - "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", - "title": "Azure Data Lake Storage REST API", - "version": "2018-06-17" - }, - "x-ms-parameterized-host": { - "hostTemplate": "{accountName}.{dnsSuffix}", - "parameters": [ - { - "$ref": "#/parameters/accountName" - }, - { - "$ref": "#/parameters/dnsSuffix" - } - ] - }, - "schemes": [ - "http", - "https" - ], - "produces": [ - "application/json" - ], - "tags": [ - { - "name": "Account Operations" - }, - { - "name": "Filesystem Operations" - }, - { - "name": "File and Directory Operations" - } - ], - "parameters": { - "Version": { - "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", - "in": "header", - "name": "x-ms-version", - "required": true, - "type": "string" - }, - "accountName": { - "description": "The Azure Storage account name.", - "in": "path", - "name": "accountName", - "required": true, - "type": "string", - "x-ms-skip-url-encoding": true - }, - "dnsSuffix": { - "default": "dfs.core.windows.net", - "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", - "in": "path", - "name": "dnsSuffix", - "required": true, - "type": "string", - "x-ms-skip-url-encoding": true - } - }, - "definitions": { - "ErrorSchema": { - "properties": { - "error": { - "description": "The service error response object.", - "properties": { - "code": { - "description": "The service error code.", - "type": "string" - }, - "message": { - "description": "The service error message.", - "type": "string" - } - } - } - } - }, - "ListEntrySchema": { - "properties": { - "name": { - "type": "string" - }, - "isDirectory": { - "default": false, - "type": "boolean" - }, - "lastModified": { - "type": "string" - }, - "eTag": { - "type": "string" - }, - "contentLength": { - "type": "integer", - "format": "int64" - }, - "owner": { - "type": "string" - }, - "group": { - "type": "string" - }, - "permissions": { - "type": "string" - } - } - }, - "ListSchema": { - "properties": { - "paths": { - "type": "array", - "items": { - "$ref": "#/definitions/ListEntrySchema" - } - } - } - }, - "ListFilesystemEntry": { - "properties": { - "name": { - "type": "string" - }, - "lastModified": { - "type": "string" - }, - "eTag": { - "type": "string" - } - } - }, - "ListFilesystemSchema": { - "properties": { - "filesystems": { - "type": "array", - "items": { - "$ref": "#/definitions/ListFilesystemEntry" - } - } - } - } - }, - "responses": { - "ErrorResponse": { - "description": "An error occurred. The possible HTTP status, code, and message strings are listed below:\n* 400 Bad Request, ContentLengthMustBeZero, \"The Content-Length request header must be zero.\"\n* 400 Bad Request, InvalidAuthenticationInfo, \"Authentication information is not given in the correct format. Check the value of Authorization header.\"\n* 400 Bad Request, InvalidFlushPosition, \"The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data.\"\n* 400 Bad Request, InvalidHeaderValue, \"The value for one of the HTTP headers is not in the correct format.\"\n* 400 Bad Request, InvalidHttpVerb, \"The HTTP verb specified is invalid - it is not recognized by the server.\"\n* 400 Bad Request, InvalidInput, \"One of the request inputs is not valid.\"\n* 400 Bad Request, InvalidPropertyName, \"A property name cannot be empty.\"\n* 400 Bad Request, InvalidPropertyName, \"The property name contains invalid characters.\"\n* 400 Bad Request, InvalidQueryParameterValue, \"Value for one of the query parameters specified in the request URI is invalid.\"\n* 400 Bad Request, InvalidResourceName, \"The specifed resource name contains invalid characters.\"\n* 400 Bad Request, InvalidSourceUri, \"The source URI is invalid.\"\n* 400 Bad Request, InvalidUri, \"The request URI is invalid.\"\n* 400 Bad Request, MissingRequiredHeader, \"An HTTP header that's mandatory for this request is not specified.\"\n* 400 Bad Request, MissingRequiredQueryParameter, \"A query parameter that's mandatory for this request is not specified.\"\n* 400 Bad Request, MultipleConditionHeadersNotSupported, \"Multiple condition headers are not supported.\"\n* 400 Bad Request, OutOfRangeInput, \"One of the request inputs is out of range.\"\n* 400 Bad Request, OutOfRangeQueryParameterValue, \"One of the query parameters specified in the request URI is outside the permissible range.\"\n* 400 Bad Request, UnsupportedHeader, \"One of the headers specified in the request is not supported.\"\n* 400 Bad Request, UnsupportedQueryParameter, \"One of the query parameters specified in the request URI is not supported.\"\n* 400 Bad Request, UnsupportedRestVersion, \"The specified Rest Version is Unsupported.\"\n* 403 Forbidden, AccountIsDisabled, \"The specified account is disabled.\"\n* 403 Forbidden, AuthorizationFailure, \"This request is not authorized to perform this operation.\"\n* 403 Forbidden, InsufficientAccountPermissions, \"The account being accessed does not have sufficient permissions to execute this operation.\"\n* 404 Not Found, FilesystemNotFound, \"The specified filesystem does not exist.\"\n* 404 Not Found, PathNotFound, \"The specified path does not exist.\"\n* 404 Not Found, RenameDestinationParentPathNotFound, \"The parent directory of the destination path does not exist.\"\n* 404 Not Found, ResourceNotFound, \"The specified resource does not exist.\"\n* 404 Not Found, SourcePathNotFound, \"The source path for a rename operation does not exist.\"\n* 405 Method Not Allowed, UnsupportedHttpVerb, \"The resource doesn't support the specified HTTP verb.\"\n* 409 Conflict, DestinationPathIsBeingDeleted, \"The specified destination path is marked to be deleted.\"\n* 409 Conflict, DirectoryNotEmpty, \"The recursive query parameter value must be true to delete a non-empty directory.\"\n* 409 Conflict, FilesystemAlreadyExists, \"The specified filesystem already exists.\"\n* 409 Conflict, FilesystemBeingDeleted, \"The specified filesystem is being deleted.\"\n* 409 Conflict, InvalidDestinationPath, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"* 409 Conflict, InvalidFlushOperation, \"The resource was created or modified by the Blob Service API and cannot be written to by the Data Lake Storage Service API.\"\n* 409 Conflict, InvalidRenameSourcePath, \"The source directory cannot be the same as the destination directory, nor can the destination be a subdirectory of the source directory.\"\n* 409 Conflict, InvalidSourceOrDestinationResourceType, \"The source and destination resource type must be identical.\"\n* 409 Conflict, LeaseAlreadyPresent, \"There is already a lease present.\"\n* 409 Conflict, LeaseIdMismatchWithLeaseOperation, \"The lease ID specified did not match the lease ID for the resource with the specified lease operation.\"\n* 409 Conflict, LeaseIsAlreadyBroken, \"The lease has already been broken and cannot be broken again.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeAcquired, \"The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeChanged, \"The lease ID matched, but the lease is currently in breaking state and cannot be changed.\"\n* 409 Conflict, LeaseIsBrokenAndCannotBeRenewed, \"The lease ID matched, but the lease has been broken explicitly and cannot be renewed.\"\n* 409 Conflict, LeaseNameMismatch, \"The lease name specified did not match the existing lease name.\"\n* 409 Conflict, LeaseNotPresentWithLeaseOperation, \"The lease ID is not present with the specified lease operation.\"\n* 409 Conflict, PathAlreadyExists, \"The specified path already exists.\"\n* 409 Conflict, PathConflict, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"\n* 409 Conflict, SourcePathIsBeingDeleted, \"The specified source path is marked to be deleted.\"\n* 409 Conflict, ResourceTypeMismatch, \"The resource type specified in the request does not match the type of the resource.\"\n* 412 Precondition Failed, ConditionNotMet, \"The condition specified using HTTP conditional header(s) is not met.\"\n* 412 Precondition Failed, LeaseIdMismatch, \"The lease ID specified did not match the lease ID for the resource.\"\n* 412 Precondition Failed, LeaseIdMissing, \"There is currently a lease on the resource and no lease ID was specified in the request.\"\n* 412 Precondition Failed, LeaseNotPresent, \"There is currently no lease on the resource.\"\n* 412 Precondition Failed, LeaseLost, \"A lease ID was specified, but the lease for the resource has expired.\"\n* 412 Precondition Failed, SourceConditionNotMet, \"The source condition specified using HTTP conditional header(s) is not met.\"\n* 413 Request Entity Too Large, RequestBodyTooLarge, \"The request body is too large and exceeds the maximum permissible limit.\"\n* 416 Requested Range Not Satisfiable, InvalidRange, \"The range specified is invalid for the current size of the resource.\"\n* 500 Internal Server Error, InternalError, \"The server encountered an internal error. Please retry the request.\"\n* 500 Internal Server Error, OperationTimedOut, \"The operation could not be completed within the permitted time.\"\n* 503 Service Unavailable, ServerBusy, \"Egress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Ingress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Operations per second is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"The server is currently unable to receive requests. Please retry your request.\"", - "headers": { - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - }, - "schema": { - "$ref": "#/definitions/ErrorSchema" - } - } - }, - "paths": { - "/": { - "get": { - "operationId": "ListFilesystems", - "summary": "List Filesystems", - "description": "List filesystems and their properties in given account.", - "tags": [ - "Account Operations" - ], - "responses": { - "200": { - "description": "OK", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", - "type": "string" - }, - "Content-Type": { - "description": "The content type of list filesystem response. The default content type is application/json.", - "type": "string" - } - }, - "schema": { - "$ref": "#/definitions/ListFilesystemSchema" - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "resource", - "in": "query", - "description": "The value must be \"account\" for all account operations.", - "required": true, - "type": "string" - }, - { - "name": "prefix", - "in": "query", - "description": "Filters results to filesystems within the specified prefix.", - "required": false, - "type": "string" - }, - { - "name": "continuation", - "in": "query", - "description": "The number of filesystems returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", - "required": false, - "type": "string" - }, - { - "name": "maxResults", - "in": "query", - "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-client-request-id", - "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", - "in": "header", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "timeout", - "in": "query", - "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-date", - "in": "header", - "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", - "required": false, - "type": "string" - }, - { - "$ref": "#/parameters/Version" - } - ] - } - }, - "/{filesystem}": { - "put": { - "operationId": "CreateFilesystem", - "summary": "Create Filesystem", - "description": "Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. This operation does not support conditional HTTP requests.", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "201": { - "description": "Created", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-namespace-enabled": { - "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "x-ms-properties", - "description": "User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "patch": { - "operationId": "SetFilesystemProperties", - "summary": "Set Filesystem Properties", - "description": "Set properties for the filesystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "x-ms-properties", - "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "get": { - "operationId": "ListPaths", - "summary": "List Paths", - "description": "List filesystem paths and their properties.", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", - "type": "string" - } - }, - "schema": { - "$ref": "#/definitions/ListSchema" - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "directory", - "in": "query", - "description": "Filters results to paths within the specified directory. An error occurs if the directory does not exist.", - "required": false, - "type": "string" - }, - { - "name": "recursive", - "in": "query", - "description": "If \"true\", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If \"directory\" is specified, the list will only include paths that share the same root.", - "required": true, - "type": "boolean" - }, - { - "name": "continuation", - "in": "query", - "description": "The number of paths returned with each invocation is limited. If the number of paths to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", - "required": false, - "type": "string" - }, - { - "name": "maxResults", - "in": "query", - "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - } - ] - }, - "head": { - "operationId": "GetFilesystemProperties", - "summary": "Get Filesystem Properties.", - "description": "All system and user-defined filesystem properties are specified in the response headers.", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-namespace-enabled": { - "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - } - }, - "delete": { - "operationId": "DeleteFilesystem", - "summary": "Delete Filesystem", - "description": "Marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "202": { - "description": "Accepted", - "headers": { - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "parameters": [ - { - "name": "filesystem", - "in": "path", - "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", - "required": true, - "type": "string" - }, - { - "name": "resource", - "in": "query", - "description": "The value must be \"filesystem\" for all filesystem operations.", - "required": true, - "type": "string" - }, - { - "name": "x-ms-client-request-id", - "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", - "in": "header", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "timeout", - "in": "query", - "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-date", - "in": "header", - "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", - "required": false, - "type": "string" - }, - { - "$ref": "#/parameters/Version" - } - ] - }, - "/{filesystem}/{path}": { - "put": { - "operationId": "CreatePath", - "summary": "Create File | Create Directory | Rename File | Rename Directory", - "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", - "consumes": [ - "application/octet-stream" - ], - "tags": [ - "File and Directory Operations" - ], - "responses": { - "201": { - "description": "The file or directory was created.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "resource", - "in": "query", - "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", - "required": false, - "type": "string" - }, - { - "name": "continuation", - "in": "query", - "description": "Optional. When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", - "required": false, - "type": "string" - }, - { - "name": "mode", - "in": "query", - "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\". ", - "required": false, - "type": "string" - }, - { - "name": "Cache-Control", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "Content-Encoding", - "in": "header", - "description": "Optional. Specifies which content encodings have been applied to the file. This value is returned to the client when the \"Read File\" operation is performed.", - "required": false, - "type": "string" - }, - { - "name": "Content-Language", - "in": "header", - "description": "Optional. Specifies the natural language used by the intended audience for the file.", - "required": false, - "type": "string" - }, - { - "name": "Content-Disposition", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-cache-control", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-type", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-encoding", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-language", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-disposition", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-rename-source", - "in": "header", - "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesysystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "Optional. A lease ID for the path specified in the URI. The path to be overwritten must have an active lease and the lease ID must match.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-proposed-lease-id", - "in": "header", - "description": "Optional for create operations. Required when \"x-ms-lease-action\" is used. A lease will be acquired using the proposed ID when the resource is created.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-lease-id", - "in": "header", - "description": "Optional for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID must match.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-properties", - "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-permissions", - "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-match", - "description": "Optional. An ETag value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-none-match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-modified-since", - "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-unmodified-since", - "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "patch": { - "operationId": "UpdatePath", - "summary": "Append Data | Flush Data | Set Properties | Set Access Control", - "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "consumes": [ - "application/octet-stream", - "text/plain" - ], - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "The data was flushed (written) to the file or the properties were set successfully.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "x-ms-properties": { - "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - } - }, - "202": { - "description": "The uploaded data was accepted.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "action", - "in": "query", - "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, or \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", - "required": true, - "type": "string" - }, - { - "name": "position", - "in": "query", - "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", - "format": "int64", - "required": false, - "type": "integer" - }, - { - "name": "retainUncommittedData", - "in": "query", - "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", - "required": false, - "type": "boolean" - }, - { - "name": "Content-Length", - "in": "header", - "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", - "minimum": 0, - "required": false, - "type": "string" - }, - { - "name": "x-ms-lease-action", - "in": "header", - "description": "Optional. The lease action can be \"renew\" to renew an existing lease or \"release\" to release a lease.", - "type": "string" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "The lease ID must be specified if there is an active lease.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-cache-control", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-type", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-disposition", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-encoding", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-language", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-properties", - "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded. Valid only for the setProperties operation. If the file or directory exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-owner", - "description": "Optional and valid only for the setAccessControl operation. Sets the owner of the file or directory.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-group", - "description": "Optional and valid only for the setAccessControl operation. Sets the owning group of the file or directory.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-permissions", - "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction with x-ms-acl.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-acl", - "description": "Optional and valid only for the setAccessControl operation. Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries that fully replaces the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\". The scope must be \"default\" to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. There are four ACE types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries. The user or group identifier is omitted for entries of type \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied. For example, the following ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning group, and nothing to everyone else: \"user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx\". Invalid in conjunction with x-ms-permissions.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-http-method-override", - "description": "Optional. Override the http verb on the service side. Some older http clients do not support PATCH", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "requestBody", - "description": "Valid only for append operations. The data to be uploaded and appended to the file.", - "in": "body", - "required": false, - "schema": { - "type": "object", - "format": "file" - } - } - ] - }, - "post": { - "operationId": "LeasePath", - "summary": "Lease Path", - "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "The \"renew\", \"change\" or \"release\" action was successful.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-lease-id": { - "description": "A successful \"renew\" action returns the lease ID.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - } - } - }, - "201": { - "description": "A new lease has been created. The \"acquire\" action was successful.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-lease-id": { - "description": "A successful \"acquire\" action returns the lease ID.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - } - } - }, - "202": { - "description": "The \"break\" lease action was successful.", - "headers": { - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-lease-time": { - "description": "The time remaining in the lease period in seconds.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "x-ms-lease-action", - "in": "header", - "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", - "required": true, - "type": "string" - }, - { - "name": "x-ms-lease-duration", - "in": "header", - "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", - "format": "int32", - "required": false, - "type": "integer" - }, - { - "name": "x-ms-lease-break-period", - "in": "header", - "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", - "format": "int32", - "required": false, - "type": "integer" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "Required when \"x-ms-lease-action\" is \"renew\", \"change\" or \"release\". For the renew and release actions, this must match the current lease ID.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-proposed-lease-id", - "in": "header", - "description": "Required when \"x-ms-lease-action\" is \"acquire\" or \"change\". A lease will be acquired with this lease ID if the operation is successful.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "get": { - "operationId": "ReadPath", - "summary": "Read File", - "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "produces": [ - "application/json", - "application/octet-stream", - "text/plain" - ], - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-resource-type": { - "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-lease-duration": { - "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", - "type": "string" - }, - "x-ms-lease-state": { - "description": "Lease state of the resource. ", - "type": "string" - }, - "x-ms-lease-status": { - "description": "The lease status of the resource.", - "type": "string" - } - }, - "schema": { - "type": "file" - } - }, - "206": { - "description": "Partial content", - "headers": { - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-resource-type": { - "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-lease-duration": { - "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", - "type": "string" - }, - "x-ms-lease-state": { - "description": "Lease state of the resource. ", - "type": "string" - }, - "x-ms-lease-status": { - "description": "The lease status of the resource.", - "type": "string" - } - }, - "schema": { - "type": "file" - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "in": "header", - "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved.", - "required": false, - "type": "string", - "name": "Range" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "head": { - "operationId": "GetPathProperties", - "summary": "Get Properties | Get Access Control List", - "description": "Get the properties for a file or directory, and optionally include the access control list. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "Returns all properties for the file or directory.", - "headers": { - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-resource-type": { - "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-owner": { - "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-group": { - "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-permissions": { - "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-acl": { - "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-lease-duration": { - "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", - "type": "string" - }, - "x-ms-lease-state": { - "description": "Lease state of the resource. ", - "type": "string" - }, - "x-ms-lease-status": { - "description": "The lease status of the resource.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "action", - "in": "query", - "description": "Optional. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account).", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "delete": { - "operationId": "DeletePath", - "summary": "Delete File | Delete Directory", - "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "The file was deleted.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "recursive", - "in": "query", - "description": "Required and valid only when the resource is a directory. If \"true\", all paths beneath the directory will be deleted. If \"false\" and the directory is non-empty, an error occurs.", - "required": false, - "type": "boolean" - }, - { - "name": "continuation", - "in": "query", - "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "The lease ID must be specified if there is an active lease.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "parameters": [ - { - "name": "filesystem", - "in": "path", - "description": "The filesystem identifier.", - "minLength": 3, - "maxLength": 63, - "required": true, - "type": "string" - }, - { - "name": "path", - "in": "path", - "description": "The file or directory path.", - "required": true, - "type": "string" - }, - { - "name": "x-ms-client-request-id", - "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", - "in": "header", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "timeout", - "in": "query", - "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-date", - "in": "header", - "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", - "required": false, - "type": "string" - }, - { - "$ref": "#/parameters/Version" - } - ] - } - } -} +{ + "swagger": "2.0", + "info": { + "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", + "title": "Azure Data Lake Storage REST API", + "version": "2018-11-09", + "x-ms-code-generation-settings": { + "internalConstructors": true, + "name": "DataLakeStorageClient" + } + }, + "x-ms-parameterized-host": { + "hostTemplate": "{accountName}.{dnsSuffix}", + "parameters": [ + { + "$ref": "#/parameters/accountName" + }, + { + "$ref": "#/parameters/dnsSuffix" + } + ] + }, + "schemes": [ + "http", + "https" + ], + "produces": [ + "application/json" + ], + "tags": [ + { + "name": "Account Operations" + }, + { + "name": "Filesystem Operations" + }, + { + "name": "File and Directory Operations" + } + ], + "parameters": { + "Version": { + "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", + "in": "header", + "name": "x-ms-version", + "required": false, + "type": "string", + "x-ms-parameter-location": "client" + }, + "accountName": { + "description": "The Azure Storage account name.", + "in": "path", + "name": "accountName", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + }, + "dnsSuffix": { + "default": "dfs.core.windows.net", + "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", + "in": "path", + "name": "dnsSuffix", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + } + }, + "definitions": { + "DataLakeStorageError": { + "properties": { + "error": { + "description": "The service error response object.", + "properties": { + "code": { + "description": "The service error code.", + "type": "string" + }, + "message": { + "description": "The service error message.", + "type": "string" + } + } + } + } + }, + "Path": { + "properties": { + "name": { + "type": "string" + }, + "isDirectory": { + "default": false, + "type": "boolean" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + }, + "contentLength": { + "type": "integer", + "format": "int64" + }, + "owner": { + "type": "string" + }, + "group": { + "type": "string" + }, + "permissions": { + "type": "string" + } + } + }, + "PathList": { + "properties": { + "paths": { + "type": "array", + "items": { + "$ref": "#/definitions/Path" + } + } + } + }, + "Filesystem": { + "properties": { + "name": { + "type": "string" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + } + } + }, + "FilesystemList": { + "properties": { + "filesystems": { + "type": "array", + "items": { + "$ref": "#/definitions/Filesystem" + } + } + } + } + }, + "responses": { + "ErrorResponse": { + "description": "An error occurred. The possible HTTP status, code, and message strings are listed below:\n* 400 Bad Request, ContentLengthMustBeZero, \"The Content-Length request header must be zero.\"\n* 400 Bad Request, InvalidAuthenticationInfo, \"Authentication information is not given in the correct format. Check the value of Authorization header.\"\n* 400 Bad Request, InvalidFlushPosition, \"The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data.\"\n* 400 Bad Request, InvalidHeaderValue, \"The value for one of the HTTP headers is not in the correct format.\"\n* 400 Bad Request, InvalidHttpVerb, \"The HTTP verb specified is invalid - it is not recognized by the server.\"\n* 400 Bad Request, InvalidInput, \"One of the request inputs is not valid.\"\n* 400 Bad Request, InvalidPropertyName, \"A property name cannot be empty.\"\n* 400 Bad Request, InvalidPropertyName, \"The property name contains invalid characters.\"\n* 400 Bad Request, InvalidQueryParameterValue, \"Value for one of the query parameters specified in the request URI is invalid.\"\n* 400 Bad Request, InvalidResourceName, \"The specified resource name contains invalid characters.\"\n* 400 Bad Request, InvalidSourceUri, \"The source URI is invalid.\"\n* 400 Bad Request, InvalidUri, \"The request URI is invalid.\"\n* 400 Bad Request, MissingRequiredHeader, \"An HTTP header that's mandatory for this request is not specified.\"\n* 400 Bad Request, MissingRequiredQueryParameter, \"A query parameter that's mandatory for this request is not specified.\"\n* 400 Bad Request, MultipleConditionHeadersNotSupported, \"Multiple condition headers are not supported.\"\n* 400 Bad Request, OutOfRangeInput, \"One of the request inputs is out of range.\"\n* 400 Bad Request, OutOfRangeQueryParameterValue, \"One of the query parameters specified in the request URI is outside the permissible range.\"\n* 400 Bad Request, UnsupportedHeader, \"One of the headers specified in the request is not supported.\"\n* 400 Bad Request, UnsupportedQueryParameter, \"One of the query parameters specified in the request URI is not supported.\"\n* 400 Bad Request, UnsupportedRestVersion, \"The specified Rest Version is Unsupported.\"\n* 403 Forbidden, AccountIsDisabled, \"The specified account is disabled.\"\n* 403 Forbidden, AuthorizationFailure, \"This request is not authorized to perform this operation.\"\n* 403 Forbidden, InsufficientAccountPermissions, \"The account being accessed does not have sufficient permissions to execute this operation.\"\n* 404 Not Found, FilesystemNotFound, \"The specified filesystem does not exist.\"\n* 404 Not Found, PathNotFound, \"The specified path does not exist.\"\n* 404 Not Found, RenameDestinationParentPathNotFound, \"The parent directory of the destination path does not exist.\"\n* 404 Not Found, ResourceNotFound, \"The specified resource does not exist.\"\n* 404 Not Found, SourcePathNotFound, \"The source path for a rename operation does not exist.\"\n* 405 Method Not Allowed, UnsupportedHttpVerb, \"The resource doesn't support the specified HTTP verb.\"\n* 409 Conflict, DestinationPathIsBeingDeleted, \"The specified destination path is marked to be deleted.\"\n* 409 Conflict, DirectoryNotEmpty, \"The recursive query parameter value must be true to delete a non-empty directory.\"\n* 409 Conflict, FilesystemAlreadyExists, \"The specified filesystem already exists.\"\n* 409 Conflict, FilesystemBeingDeleted, \"The specified filesystem is being deleted.\"\n* 409 Conflict, InvalidDestinationPath, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"* 409 Conflict, InvalidFlushOperation, \"The resource was created or modified by the Blob Service API and cannot be written to by the Data Lake Storage Service API.\"\n* 409 Conflict, InvalidRenameSourcePath, \"The source directory cannot be the same as the destination directory, nor can the destination be a subdirectory of the source directory.\"\n* 409 Conflict, InvalidSourceOrDestinationResourceType, \"The source and destination resource type must be identical.\"\n* 409 Conflict, LeaseAlreadyPresent, \"There is already a lease present.\"\n* 409 Conflict, LeaseIdMismatchWithLeaseOperation, \"The lease ID specified did not match the lease ID for the resource with the specified lease operation.\"\n* 409 Conflict, LeaseIsAlreadyBroken, \"The lease has already been broken and cannot be broken again.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeAcquired, \"The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeChanged, \"The lease ID matched, but the lease is currently in breaking state and cannot be changed.\"\n* 409 Conflict, LeaseIsBrokenAndCannotBeRenewed, \"The lease ID matched, but the lease has been broken explicitly and cannot be renewed.\"\n* 409 Conflict, LeaseNameMismatch, \"The lease name specified did not match the existing lease name.\"\n* 409 Conflict, LeaseNotPresentWithLeaseOperation, \"The lease ID is not present with the specified lease operation.\"\n* 409 Conflict, PathAlreadyExists, \"The specified path already exists.\"\n* 409 Conflict, PathConflict, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"\n* 409 Conflict, SourcePathIsBeingDeleted, \"The specified source path is marked to be deleted.\"\n* 409 Conflict, ResourceTypeMismatch, \"The resource type specified in the request does not match the type of the resource.\"\n* 412 Precondition Failed, ConditionNotMet, \"The condition specified using HTTP conditional header(s) is not met.\"\n* 412 Precondition Failed, LeaseIdMismatch, \"The lease ID specified did not match the lease ID for the resource.\"\n* 412 Precondition Failed, LeaseIdMissing, \"There is currently a lease on the resource and no lease ID was specified in the request.\"\n* 412 Precondition Failed, LeaseNotPresent, \"There is currently no lease on the resource.\"\n* 412 Precondition Failed, LeaseLost, \"A lease ID was specified, but the lease for the resource has expired.\"\n* 412 Precondition Failed, SourceConditionNotMet, \"The source condition specified using HTTP conditional header(s) is not met.\"\n* 413 Request Entity Too Large, RequestBodyTooLarge, \"The request body is too large and exceeds the maximum permissible limit.\"\n* 416 Requested Range Not Satisfiable, InvalidRange, \"The range specified is invalid for the current size of the resource.\"\n* 500 Internal Server Error, InternalError, \"The server encountered an internal error. Please retry the request.\"\n* 500 Internal Server Error, OperationTimedOut, \"The operation could not be completed within the permitted time.\"\n* 503 Service Unavailable, ServerBusy, \"Egress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Ingress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Operations per second is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"The server is currently unable to receive requests. Please retry your request.\"", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/DataLakeStorageError" + } + } + }, + "paths": { + "/": { + "get": { + "operationId": "Filesystem_List", + "summary": "List Filesystems", + "description": "List filesystems and their properties in given account.", + "x-ms-pageable": { + "itemName": "filesystems", + "nextLinkName": null + }, + "tags": [ + "Account Operations" + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "type": "string" + }, + "Content-Type": { + "description": "The content type of list filesystem response. The default content type is application/json.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/FilesystemList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "The value must be \"account\" for all account operations.", + "required": true, + "type": "string", + "enum": [ + "account" + ], + "x-ms-enum": { + "name": "AccountResourceType", + "modelAsString": false + } + }, + { + "name": "prefix", + "in": "query", + "description": "Filters results to filesystems within the specified prefix.", + "required": false, + "type": "string" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of filesystems returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + }, + "/{filesystem}": { + "put": { + "operationId": "Filesystem_Create", + "summary": "Create Filesystem", + "description": "Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. This operation does not support conditional HTTP requests.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Filesystem_SetProperties", + "summary": "Set Filesystem Properties", + "description": "Set properties for the filesystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_List", + "summary": "List Paths", + "description": "List filesystem paths and their properties.", + "x-ms-pageable": { + "itemName": "paths", + "nextLinkName": null + }, + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/PathList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "directory", + "in": "query", + "description": "Filters results to paths within the specified directory. An error occurs if the directory does not exist.", + "required": false, + "type": "string" + }, + { + "name": "recursive", + "in": "query", + "description": "If \"true\", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If \"directory\" is specified, the list will only include paths that share the same root.", + "required": true, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of paths returned with each invocation is limited. If the number of paths to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + } + ] + }, + "head": { + "operationId": "Filesystem_GetProperties", + "summary": "Get Filesystem Properties.", + "description": "All system and user-defined filesystem properties are specified in the response headers.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + } + }, + "delete": { + "operationId": "Filesystem_Delete", + "summary": "Delete Filesystem", + "description": "Marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "resource", + "in": "query", + "description": "The value must be \"filesystem\" for all filesystem operations.", + "required": true, + "type": "string", + "enum": [ + "filesystem" + ], + "x-ms-enum": { + "name": "FilesystemResourceType", + "modelAsString": false + } + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + }, + "/{filesystem}/{path}": { + "put": { + "operationId": "Path_Create", + "summary": "Create File | Create Directory | Rename File | Rename Directory", + "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", + "consumes": [ + "application/octet-stream" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "201": { + "description": "The file or directory was created.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", + "required": false, + "type": "string", + "enum": [ + "directory", + "file" + ], + "x-ms-enum": { + "name": "PathResourceType", + "modelAsString": false + } + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "required": false, + "type": "string" + }, + { + "name": "mode", + "in": "query", + "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\". ", + "required": false, + "type": "string", + "enum": [ + "legacy", + "posix" + ], + "x-ms-enum": { + "name": "PathRenameMode", + "modelAsString": false + } + }, + { + "name": "Cache-Control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "Content-Encoding", + "in": "header", + "description": "Optional. Specifies which content encodings have been applied to the file. This value is returned to the client when the \"Read File\" operation is performed.", + "required": false, + "type": "string" + }, + { + "name": "Content-Language", + "in": "header", + "description": "Optional. Specifies the natural language used by the intended audience for the file.", + "required": false, + "type": "string" + }, + { + "name": "Content-Disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-rename-source", + "in": "header", + "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. A lease ID for the path specified in the URI. The path to be overwritten must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-lease-id", + "in": "header", + "description": "Optional for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-umask", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation (e.g. 0766).", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-match", + "description": "Optional. An ETag value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-none-match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-modified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-unmodified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Path_Update", + "summary": "Append Data | Flush Data | Set Properties | Set Access Control", + "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "consumes": [ + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The data was flushed (written) to the file or the properties were set successfully.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "x-ms-properties": { + "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "202": { + "description": "The uploaded data was accepted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, or \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", + "required": true, + "type": "string", + "enum": [ + "append", + "flush", + "setProperties", + "setAccessControl" + ], + "x-ms-enum": { + "name": "PathUpdateAction", + "modelAsString": false + } + }, + { + "name": "position", + "in": "query", + "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", + "format": "int64", + "required": false, + "type": "integer" + }, + { + "name": "retainUncommittedData", + "in": "query", + "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", + "required": false, + "type": "boolean" + }, + { + "name": "close", + "in": "query", + "description": "Azure Storage Events allow applications to receive notifications when files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property indicating whether this is the final change to distinguish the difference between an intermediate flush to a file stream and the final close of a file stream. The close query parameter is valid only when the action is \"flush\" and change notifications are enabled. If the value of close is \"true\" and the flush operation completes successfully, the service raises a file change notification with a property indicating that this is the final update (the file stream has been closed). If \"false\" a change notification is raised indicating the file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been closed.\"", + "required": false, + "type": "boolean" + }, + { + "name": "Content-Length", + "in": "header", + "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", + "minimum": 0, + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-md5", + "in": "header", + "description": "Optional and only valid for \"Flush & Set Properties\" operations. The service stores this value and includes it in the \"Content-Md5\" response header for \"Read & Get Properties\" operations. If this property is not specified on the request, then the property will be cleared for the file. Subsequent calls to \"Read & Get Properties\" will not return this property unless it is explicitly set on that file again.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. Valid only for the setProperties operation. If the file or directory exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-owner", + "description": "Optional and valid only for the setAccessControl operation. Sets the owner of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-group", + "description": "Optional and valid only for the setAccessControl operation. Sets the owning group of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction with x-ms-acl.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-acl", + "description": "Optional and valid only for the setAccessControl operation. Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries that fully replaces the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\". The scope must be \"default\" to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. There are four ACE types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries. The user or group identifier is omitted for entries of type \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied. For example, the following ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning group, and nothing to everyone else: \"user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx\". Invalid in conjunction with x-ms-permissions.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "requestBody", + "description": "Valid only for append operations. The data to be uploaded and appended to the file.", + "in": "body", + "required": false, + "schema": { + "type": "object", + "format": "file" + } + } + ] + }, + "post": { + "operationId": "Path_Lease", + "summary": "Lease Path", + "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The \"renew\", \"change\" or \"release\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"renew\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "201": { + "description": "A new lease has been created. The \"acquire\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"acquire\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "202": { + "description": "The \"break\" lease action was successful.", + "headers": { + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-time": { + "description": "The time remaining in the lease period in seconds.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-lease-action", + "in": "header", + "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", + "required": true, + "type": "string", + "enum": [ + "acquire", + "break", + "change", + "renew", + "release" + ], + "x-ms-enum": { + "name": "PathLeaseAction", + "modelAsString": false + } + }, + { + "name": "x-ms-lease-duration", + "in": "header", + "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-break-period", + "in": "header", + "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"renew\", \"change\" or \"release\". For the renew and release actions, this must match the current lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-proposed-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"acquire\" or \"change\". A lease will be acquired with this lease ID if the operation is successful.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_Read", + "summary": "Read File", + "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "produces": [ + "application/json", + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file. If the file has an MD5 hash and this read operation is to read the complete file, this response header is returned so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "206": { + "description": "Partial content", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "in": "header", + "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved.", + "required": false, + "type": "string", + "name": "Range" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "head": { + "operationId": "Path_GetProperties", + "summary": "Get Properties | Get Status | Get Access Control List", + "description": "Get Properties returns all system and user defined properties for a path. Get Status returns all system defined properties for a path. Get Access Control List returns the access control list for a path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Returns all properties for the file or directory.", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file stored in storage. This header is returned only for \"GetProperties\" operation. If the Content-MD5 header has been set for the file, this response header is returned for GetProperties call so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-owner": { + "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-group": { + "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-permissions": { + "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-acl": { + "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "Optional. If the value is \"getStatus\" only the system defined properties for the path are returned. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account), otherwise the properties are returned.", + "required": false, + "type": "string", + "enum": [ + "getAccessControl", + "getStatus" + ], + "x-ms-enum": { + "name": "PathGetPropertiesAction", + "modelAsString": false + } + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "delete": { + "operationId": "Path_Delete", + "summary": "Delete File | Delete Directory", + "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The file was deleted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "recursive", + "in": "query", + "description": "Required and valid only when the resource is a directory. If \"true\", all paths beneath the directory will be deleted. If \"false\" and the directory is non-empty, an error occurs.", + "required": false, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "path", + "in": "path", + "description": "The file or directory path.", + "required": true, + "type": "string" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + } +} \ No newline at end of file diff --git a/azbfs/azure_dfs_swagger_manually_edited.json b/azbfs/azure_dfs_swagger_manually_edited.json new file mode 100644 index 000000000..62be5c3b2 --- /dev/null +++ b/azbfs/azure_dfs_swagger_manually_edited.json @@ -0,0 +1,1891 @@ +{ + "swagger": "2.0", + "info": { + "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", + "title": "Azure Data Lake Storage REST API", + "version": "2018-11-09", + "x-ms-code-generation-settings": { + "internalConstructors": true, + "name": "DataLakeStorageClient" + } + }, + "x-ms-parameterized-host": { + "hostTemplate": "{accountName}.{dnsSuffix}", + "parameters": [ + { + "$ref": "#/parameters/accountName" + }, + { + "$ref": "#/parameters/dnsSuffix" + } + ] + }, + "schemes": [ + "http", + "https" + ], + "produces": [ + "application/json" + ], + "tags": [ + { + "name": "Account Operations" + }, + { + "name": "Filesystem Operations" + }, + { + "name": "File and Directory Operations" + } + ], + "parameters": { + "Version": { + "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", + "in": "header", + "name": "x-ms-version", + "required": true, + "type": "string" + }, + "accountName": { + "description": "The Azure Storage account name.", + "in": "path", + "name": "accountName", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + }, + "dnsSuffix": { + "default": "dfs.core.windows.net", + "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", + "in": "path", + "name": "dnsSuffix", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + } + }, + "definitions": { + "DataLakeStorageError": { + "properties": { + "error": { + "description": "The service error response object.", + "properties": { + "code": { + "description": "The service error code.", + "type": "string" + }, + "message": { + "description": "The service error message.", + "type": "string" + } + } + } + } + }, + "Path": { + "properties": { + "name": { + "type": "string" + }, + "isDirectory": { + "default": false, + "type": "boolean" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + }, + "contentLength": { + "type": "integer", + "format": "int64" + }, + "owner": { + "type": "string" + }, + "group": { + "type": "string" + }, + "permissions": { + "type": "string" + } + } + }, + "PathList": { + "properties": { + "paths": { + "type": "array", + "items": { + "$ref": "#/definitions/Path" + } + } + } + }, + "Filesystem": { + "properties": { + "name": { + "type": "string" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + } + } + }, + "FilesystemList": { + "properties": { + "filesystems": { + "type": "array", + "items": { + "$ref": "#/definitions/Filesystem" + } + } + } + } + }, + "responses": { + "ErrorResponse": { + "description": "An error occurred. The possible HTTP status, code, and message strings are listed below:\n* 400 Bad Request, ContentLengthMustBeZero, \"The Content-Length request header must be zero.\"\n* 400 Bad Request, InvalidAuthenticationInfo, \"Authentication information is not given in the correct format. Check the value of Authorization header.\"\n* 400 Bad Request, InvalidFlushPosition, \"The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data.\"\n* 400 Bad Request, InvalidHeaderValue, \"The value for one of the HTTP headers is not in the correct format.\"\n* 400 Bad Request, InvalidHttpVerb, \"The HTTP verb specified is invalid - it is not recognized by the server.\"\n* 400 Bad Request, InvalidInput, \"One of the request inputs is not valid.\"\n* 400 Bad Request, InvalidPropertyName, \"A property name cannot be empty.\"\n* 400 Bad Request, InvalidPropertyName, \"The property name contains invalid characters.\"\n* 400 Bad Request, InvalidQueryParameterValue, \"Value for one of the query parameters specified in the request URI is invalid.\"\n* 400 Bad Request, InvalidResourceName, \"The specified resource name contains invalid characters.\"\n* 400 Bad Request, InvalidSourceUri, \"The source URI is invalid.\"\n* 400 Bad Request, InvalidUri, \"The request URI is invalid.\"\n* 400 Bad Request, MissingRequiredHeader, \"An HTTP header that's mandatory for this request is not specified.\"\n* 400 Bad Request, MissingRequiredQueryParameter, \"A query parameter that's mandatory for this request is not specified.\"\n* 400 Bad Request, MultipleConditionHeadersNotSupported, \"Multiple condition headers are not supported.\"\n* 400 Bad Request, OutOfRangeInput, \"One of the request inputs is out of range.\"\n* 400 Bad Request, OutOfRangeQueryParameterValue, \"One of the query parameters specified in the request URI is outside the permissible range.\"\n* 400 Bad Request, UnsupportedHeader, \"One of the headers specified in the request is not supported.\"\n* 400 Bad Request, UnsupportedQueryParameter, \"One of the query parameters specified in the request URI is not supported.\"\n* 400 Bad Request, UnsupportedRestVersion, \"The specified Rest Version is Unsupported.\"\n* 403 Forbidden, AccountIsDisabled, \"The specified account is disabled.\"\n* 403 Forbidden, AuthorizationFailure, \"This request is not authorized to perform this operation.\"\n* 403 Forbidden, InsufficientAccountPermissions, \"The account being accessed does not have sufficient permissions to execute this operation.\"\n* 404 Not Found, FilesystemNotFound, \"The specified filesystem does not exist.\"\n* 404 Not Found, PathNotFound, \"The specified path does not exist.\"\n* 404 Not Found, RenameDestinationParentPathNotFound, \"The parent directory of the destination path does not exist.\"\n* 404 Not Found, ResourceNotFound, \"The specified resource does not exist.\"\n* 404 Not Found, SourcePathNotFound, \"The source path for a rename operation does not exist.\"\n* 405 Method Not Allowed, UnsupportedHttpVerb, \"The resource doesn't support the specified HTTP verb.\"\n* 409 Conflict, DestinationPathIsBeingDeleted, \"The specified destination path is marked to be deleted.\"\n* 409 Conflict, DirectoryNotEmpty, \"The recursive query parameter value must be true to delete a non-empty directory.\"\n* 409 Conflict, FilesystemAlreadyExists, \"The specified filesystem already exists.\"\n* 409 Conflict, FilesystemBeingDeleted, \"The specified filesystem is being deleted.\"\n* 409 Conflict, InvalidDestinationPath, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"* 409 Conflict, InvalidFlushOperation, \"The resource was created or modified by the Blob Service API and cannot be written to by the Data Lake Storage Service API.\"\n* 409 Conflict, InvalidRenameSourcePath, \"The source directory cannot be the same as the destination directory, nor can the destination be a subdirectory of the source directory.\"\n* 409 Conflict, InvalidSourceOrDestinationResourceType, \"The source and destination resource type must be identical.\"\n* 409 Conflict, LeaseAlreadyPresent, \"There is already a lease present.\"\n* 409 Conflict, LeaseIdMismatchWithLeaseOperation, \"The lease ID specified did not match the lease ID for the resource with the specified lease operation.\"\n* 409 Conflict, LeaseIsAlreadyBroken, \"The lease has already been broken and cannot be broken again.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeAcquired, \"The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeChanged, \"The lease ID matched, but the lease is currently in breaking state and cannot be changed.\"\n* 409 Conflict, LeaseIsBrokenAndCannotBeRenewed, \"The lease ID matched, but the lease has been broken explicitly and cannot be renewed.\"\n* 409 Conflict, LeaseNameMismatch, \"The lease name specified did not match the existing lease name.\"\n* 409 Conflict, LeaseNotPresentWithLeaseOperation, \"The lease ID is not present with the specified lease operation.\"\n* 409 Conflict, PathAlreadyExists, \"The specified path already exists.\"\n* 409 Conflict, PathConflict, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"\n* 409 Conflict, SourcePathIsBeingDeleted, \"The specified source path is marked to be deleted.\"\n* 409 Conflict, ResourceTypeMismatch, \"The resource type specified in the request does not match the type of the resource.\"\n* 412 Precondition Failed, ConditionNotMet, \"The condition specified using HTTP conditional header(s) is not met.\"\n* 412 Precondition Failed, LeaseIdMismatch, \"The lease ID specified did not match the lease ID for the resource.\"\n* 412 Precondition Failed, LeaseIdMissing, \"There is currently a lease on the resource and no lease ID was specified in the request.\"\n* 412 Precondition Failed, LeaseNotPresent, \"There is currently no lease on the resource.\"\n* 412 Precondition Failed, LeaseLost, \"A lease ID was specified, but the lease for the resource has expired.\"\n* 412 Precondition Failed, SourceConditionNotMet, \"The source condition specified using HTTP conditional header(s) is not met.\"\n* 413 Request Entity Too Large, RequestBodyTooLarge, \"The request body is too large and exceeds the maximum permissible limit.\"\n* 416 Requested Range Not Satisfiable, InvalidRange, \"The range specified is invalid for the current size of the resource.\"\n* 500 Internal Server Error, InternalError, \"The server encountered an internal error. Please retry the request.\"\n* 500 Internal Server Error, OperationTimedOut, \"The operation could not be completed within the permitted time.\"\n* 503 Service Unavailable, ServerBusy, \"Egress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Ingress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Operations per second is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"The server is currently unable to receive requests. Please retry your request.\"", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/DataLakeStorageError" + } + } + }, + "paths": { + "/": { + "get": { + "operationId": "Filesystem_List", + "summary": "List Filesystems", + "description": "List filesystems and their properties in given account.", + "x-ms-pageable": { + "itemName": "filesystems", + "nextLinkName": null + }, + "tags": [ + "Account Operations" + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "type": "string" + }, + "Content-Type": { + "description": "The content type of list filesystem response. The default content type is application/json.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/FilesystemList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "The value must be \"account\" for all account operations.", + "required": true, + "type": "string", + "enum": [ + "account" + ], + "x-ms-enum": { + "name": "AccountResourceType", + "modelAsString": false + } + }, + { + "name": "prefix", + "in": "query", + "description": "Filters results to filesystems within the specified prefix.", + "required": false, + "type": "string" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of filesystems returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + }, + "/{filesystem}": { + "put": { + "operationId": "Filesystem_Create", + "summary": "Create Filesystem", + "description": "Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. This operation does not support conditional HTTP requests.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Filesystem_SetProperties", + "summary": "Set Filesystem Properties", + "description": "Set properties for the filesystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Filesystem_ListPaths", + "summary": "List Paths", + "description": "List filesystem paths and their properties.", + "x-ms-pageable": { + "itemName": "paths", + "nextLinkName": null + }, + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/PathList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "directory", + "in": "query", + "description": "Filters results to paths within the specified directory. An error occurs if the directory does not exist.", + "required": false, + "type": "string" + }, + { + "name": "recursive", + "in": "query", + "description": "If \"true\", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If \"directory\" is specified, the list will only include paths that share the same root.", + "required": true, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of paths returned with each invocation is limited. If the number of paths to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + } + ] + }, + "head": { + "operationId": "Filesystem_GetProperties", + "summary": "Get Filesystem Properties.", + "description": "All system and user-defined filesystem properties are specified in the response headers.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + } + }, + "delete": { + "operationId": "Filesystem_Delete", + "summary": "Delete Filesystem", + "description": "Marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", + "pattern": "^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "resource", + "in": "query", + "description": "The value must be \"filesystem\" for all filesystem operations.", + "required": true, + "type": "string", + "enum": [ + "filesystem" + ], + "x-ms-enum": { + "name": "FilesystemResourceType", + "modelAsString": false + } + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + }, + "/{filesystem}/{path}": { + "put": { + "operationId": "Path_Create", + "summary": "Create File | Create Directory | Rename File | Rename Directory", + "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", + "consumes": [ + "application/octet-stream" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "201": { + "description": "The file or directory was created.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", + "required": false, + "type": "string", + "enum": [ + "directory", + "file" + ], + "x-ms-enum": { + "name": "PathResourceType", + "modelAsString": false + } + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "required": false, + "type": "string" + }, + { + "name": "mode", + "in": "query", + "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\". ", + "required": false, + "type": "string", + "enum": [ + "legacy", + "posix" + ], + "x-ms-enum": { + "name": "PathRenameMode", + "modelAsString": false + } + }, + { + "name": "Cache-Control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "Content-Encoding", + "in": "header", + "description": "Optional. Specifies which content encodings have been applied to the file. This value is returned to the client when the \"Read File\" operation is performed.", + "required": false, + "type": "string" + }, + { + "name": "Content-Language", + "in": "header", + "description": "Optional. Specifies the natural language used by the intended audience for the file.", + "required": false, + "type": "string" + }, + { + "name": "Content-Disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-rename-source", + "in": "header", + "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. A lease ID for the path specified in the URI. The path to be overwritten must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-lease-id", + "in": "header", + "description": "Optional for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-umask", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation (e.g. 0766).", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-match", + "description": "Optional. An ETag value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-none-match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-modified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-unmodified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Path_Update", + "summary": "Append Data | Flush Data | Set Properties | Set Access Control", + "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "consumes": [ + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The data was flushed (written) to the file or the properties were set successfully.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "x-ms-properties": { + "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "202": { + "description": "The uploaded data was accepted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, or \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", + "required": true, + "type": "string", + "enum": [ + "append", + "flush", + "setProperties", + "setAccessControl" + ], + "x-ms-enum": { + "name": "PathUpdateAction", + "modelAsString": false + } + }, + { + "name": "position", + "in": "query", + "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", + "format": "int64", + "required": false, + "type": "integer" + }, + { + "name": "retainUncommittedData", + "in": "query", + "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", + "required": false, + "type": "boolean" + }, + { + "name": "close", + "in": "query", + "description": "Azure Storage Events allow applications to receive notifications when files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property indicating whether this is the final change to distinguish the difference between an intermediate flush to a file stream and the final close of a file stream. The close query parameter is valid only when the action is \"flush\" and change notifications are enabled. If the value of close is \"true\" and the flush operation completes successfully, the service raises a file change notification with a property indicating that this is the final update (the file stream has been closed). If \"false\" a change notification is raised indicating the file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been closed.\"", + "required": false, + "type": "boolean" + }, + { + "name": "Content-Length", + "in": "header", + "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", + "minimum": 0, + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-md5", + "in": "header", + "description": "Optional and only valid for \"Flush & Set Properties\" operations. The service stores this value and includes it in the \"Content-Md5\" response header for \"Read & Get Properties\" operations. If this property is not specified on the request, then the property will be cleared for the file. Subsequent calls to \"Read & Get Properties\" will not return this property unless it is explicitly set on that file again.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. Valid only for the setProperties operation. If the file or directory exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-owner", + "description": "Optional and valid only for the setAccessControl operation. Sets the owner of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-group", + "description": "Optional and valid only for the setAccessControl operation. Sets the owning group of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction with x-ms-acl.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-acl", + "description": "Optional and valid only for the setAccessControl operation. Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries that fully replaces the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\". The scope must be \"default\" to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. There are four ACE types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries. The user or group identifier is omitted for entries of type \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied. For example, the following ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning group, and nothing to everyone else: \"user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx\". Invalid in conjunction with x-ms-permissions.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-http-method-override", + "description": "Optional. Override the http verb on the service side. Some older http clients do not support PATCH", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "requestBody", + "description": "Valid only for append operations. The data to be uploaded and appended to the file.", + "in": "body", + "required": false, + "schema": { + "type": "object", + "format": "file" + } + } + ] + }, + "post": { + "operationId": "Path_Lease", + "summary": "Lease Path", + "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The \"renew\", \"change\" or \"release\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"renew\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "201": { + "description": "A new lease has been created. The \"acquire\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"acquire\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "202": { + "description": "The \"break\" lease action was successful.", + "headers": { + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-time": { + "description": "The time remaining in the lease period in seconds.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-lease-action", + "in": "header", + "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", + "required": true, + "type": "string", + "enum": [ + "acquire", + "break", + "change", + "renew", + "release" + ], + "x-ms-enum": { + "name": "PathLeaseAction", + "modelAsString": false + } + }, + { + "name": "x-ms-lease-duration", + "in": "header", + "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-break-period", + "in": "header", + "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"renew\", \"change\" or \"release\". For the renew and release actions, this must match the current lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-proposed-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"acquire\" or \"change\". A lease will be acquired with this lease ID if the operation is successful.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_Read", + "summary": "Read File", + "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "produces": [ + "application/json", + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file. If the file has an MD5 hash and this read operation is to read the complete file, this response header is returned so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "206": { + "description": "Partial content", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "in": "header", + "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved.", + "required": false, + "type": "string", + "name": "Range" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "head": { + "operationId": "Path_GetProperties", + "summary": "Get Properties | Get Status | Get Access Control List", + "description": "Get Properties returns all system and user defined properties for a path. Get Status returns all system defined properties for a path. Get Access Control List returns the access control list for a path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Returns all properties for the file or directory.", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file stored in storage. This header is returned only for \"GetProperties\" operation. If the Content-MD5 header has been set for the file, this response header is returned for GetProperties call so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-owner": { + "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-group": { + "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-permissions": { + "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-acl": { + "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "Optional. If the value is \"getStatus\" only the system defined properties for the path are returned. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account), otherwise the properties are returned.", + "required": false, + "type": "string", + "enum": [ + "getAccessControl", + "getStatus" + ], + "x-ms-enum": { + "name": "PathGetPropertiesAction", + "modelAsString": false + } + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "delete": { + "operationId": "Path_Delete", + "summary": "Delete File | Delete Directory", + "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The file was deleted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "recursive", + "in": "query", + "description": "Required and valid only when the resource is a directory. If \"true\", all paths beneath the directory will be deleted. If \"false\" and the directory is non-empty, an error occurs.", + "required": false, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier.", + "pattern": "^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "path", + "in": "path", + "description": "The file or directory path.", + "required": true, + "type": "string" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + } +} diff --git a/azbfs/url_directory.go b/azbfs/url_directory.go index b7573d971..997e044e7 100644 --- a/azbfs/url_directory.go +++ b/azbfs/url_directory.go @@ -11,7 +11,7 @@ var directoryResourceName = "directory" // constant value for the resource query // A DirectoryURL represents a URL to the Azure Storage directory allowing you to manipulate its directories and files. type DirectoryURL struct { - directoryClient managementClient + directoryClient pathClient // filesystem is the filesystem identifier filesystem string // pathParameter is the file or directory path @@ -24,7 +24,7 @@ func NewDirectoryURL(url url.URL, p pipeline.Pipeline) DirectoryURL { panic("p can't be nil") } urlParts := NewBfsURLParts(url) - directoryClient := newManagementClient(url, p) + directoryClient := newPathClient(url, p) return DirectoryURL{directoryClient: directoryClient, filesystem: urlParts.FileSystemName, pathParameter: urlParts.DirectoryOrFilePath} } @@ -65,8 +65,8 @@ func (d DirectoryURL) NewDirectoryURL(dirName string) DirectoryURL { // Create creates a new directory within a File System func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, error) { - resp, err := d.directoryClient.CreatePath(ctx, d.filesystem, d.pathParameter, &directoryResourceName, nil, - nil, nil, nil, nil, nil, nil, + resp, err := d.directoryClient.Create(ctx, d.filesystem, d.pathParameter, PathResourceDirectory, nil, + PathRenameModeNone, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, @@ -77,15 +77,19 @@ func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, err // Delete removes the specified empty directory. Note that the directory must be empty before it can be deleted.. // For more information, see https://docs.microsoft.com/rest/api/storageservices/delete-directory. func (d DirectoryURL) Delete(ctx context.Context, continuationString *string, recursive bool) (*DirectoryDeleteResponse, error) { - resp, err := d.directoryClient.DeletePath(ctx, d.filesystem, d.pathParameter, &recursive, continuationString, nil, + resp, err := d.directoryClient.Delete(ctx, d.filesystem, d.pathParameter, &recursive, continuationString, nil, nil, nil, nil, nil, nil, nil, nil) return (*DirectoryDeleteResponse)(resp), err } // GetProperties returns the directory's metadata and system properties. func (d DirectoryURL) GetProperties(ctx context.Context) (*DirectoryGetPropertiesResponse, error) { - resp, err := d.directoryClient.GetPathProperties(ctx, d.filesystem, d.pathParameter, nil, nil, nil, - nil, nil, nil, nil, nil) + // Action MUST be "none", not "getStatus" because the latter does not include the MD5, and + // sometimes we call this method on things that are actually files + action := PathGetPropertiesActionNone + + resp, err := d.directoryClient.GetProperties(ctx, d.filesystem, d.pathParameter, action, nil, nil, + nil, nil, nil, nil, nil, nil, nil) return (*DirectoryGetPropertiesResponse)(resp), err } @@ -106,12 +110,13 @@ func (d DirectoryURL) FileSystemURL() FileSystemURL { // Marker) to get the next segment. func (d DirectoryURL) ListDirectorySegment(ctx context.Context, marker *string, recursive bool) (*DirectoryListResponse, error) { // Since listPath is supported on filesystem Url - // covert the directory url to fileSystemUrl + // convert the directory url to fileSystemUrl // and listPath for filesystem with directory path set in the path parameter var maxEntriesInListOperation = int32(1000) - resp, err := d.FileSystemURL().fileSystemClient.ListPaths(ctx, recursive, d.filesystem, fileSystemResourceName, &d.pathParameter, marker, - &maxEntriesInListOperation, nil, nil, nil) + resp, err := d.FileSystemURL().fileSystemClient.ListPaths(ctx, recursive, d.filesystem, &d.pathParameter, marker, + &maxEntriesInListOperation, nil, nil, nil, nil) + return (*DirectoryListResponse)(resp), err } diff --git a/azbfs/url_file.go b/azbfs/url_file.go index ffe6c4ce3..3c9e79614 100644 --- a/azbfs/url_file.go +++ b/azbfs/url_file.go @@ -2,17 +2,17 @@ package azbfs import ( "context" + "encoding/base64" "net/url" "github.com/Azure/azure-pipeline-go/pipeline" "io" "net/http" - "strconv" ) // A FileURL represents a URL to an Azure Storage file. type FileURL struct { - fileClient managementClient + fileClient pathClient fileSystemName string path string } @@ -22,7 +22,7 @@ func NewFileURL(url url.URL, p pipeline.Pipeline) FileURL { if p == nil { panic("p can't be nil") } - fileClient := newManagementClient(url, p) + fileClient := newPathClient(url, p) urlParts := NewBfsURLParts(url) return FileURL{fileClient: fileClient, fileSystemName: urlParts.FileSystemName, path: urlParts.DirectoryOrFilePath} @@ -46,10 +46,9 @@ func (f FileURL) WithPipeline(p pipeline.Pipeline) FileURL { // Create creates a new file or replaces a file. Note that this method only initializes the file. // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/create-file. -func (f FileURL) Create(ctx context.Context) (*CreatePathResponse, error) { - fileType := "file" - return f.fileClient.CreatePath(ctx, f.fileSystemName, f.path, &fileType, - nil, nil, nil, nil, nil, nil, +func (f FileURL) Create(ctx context.Context) (*PathCreateResponse, error) { + return f.fileClient.Create(ctx, f.fileSystemName, f.path, PathResourceFile, + nil, PathRenameModeNone, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, @@ -62,8 +61,8 @@ func (f FileURL) Create(ctx context.Context) (*CreatePathResponse, error) { // response header/property if the range is <= 4MB; the HTTP request fails with 400 (Bad Request) if the requested range is greater than 4MB. // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file. func (f FileURL) Download(ctx context.Context, offset int64, count int64) (*DownloadResponse, error) { - dr, err := f.fileClient.ReadPath(ctx, f.fileSystemName, f.path, (&httpRange{offset: offset, count: count}).pointers(), - nil, nil, nil, nil, nil, nil, nil) + dr, err := f.fileClient.Read(ctx, f.fileSystemName, f.path, (&httpRange{offset: offset, count: count}).pointers(), + nil, nil, nil, nil, nil, nil, nil, nil) if err != nil { return nil, err } @@ -72,7 +71,9 @@ func (f FileURL) Download(ctx context.Context, offset int64, count int64) (*Down f: f, dr: dr, ctx: ctx, - info: HTTPGetterInfo{Offset: offset, Count: count, ETag: dr.ETag()}, // TODO: Note conditional header is not currently supported in Azure File. + info: HTTPGetterInfo{Offset: offset, Count: count, ETag: dr.ETag()}, + // TODO: Note conditional header is not currently supported in Azure File. + // TODO: review the above todo, since as of 8 Feb 2019 we are on a newer version of the API }, err } @@ -99,24 +100,28 @@ func (dr *DownloadResponse) Body(o RetryReaderOptions) io.ReadCloser { // Delete immediately removes the file from the storage account. // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/delete-file2. -func (f FileURL) Delete(ctx context.Context) (*DeletePathResponse, error) { +func (f FileURL) Delete(ctx context.Context) (*PathDeleteResponse, error) { recursive := false - return f.fileClient.DeletePath(ctx, f.fileSystemName, f.path, &recursive, + return f.fileClient.Delete(ctx, f.fileSystemName, f.path, &recursive, nil, nil, nil, nil, nil, nil, nil, nil, nil) } // GetProperties returns the file's metadata and properties. // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file-properties. -func (f FileURL) GetProperties(ctx context.Context) (*GetPathPropertiesResponse, error) { - return f.fileClient.GetPathProperties(ctx, f.fileSystemName, f.path, nil, nil, +func (f FileURL) GetProperties(ctx context.Context) (*PathGetPropertiesResponse, error) { + // Action MUST be "none", not "getStatus" because the latter does not include the MD5, and + // sometimes we call this method on things that are actually files + action := PathGetPropertiesActionNone + + return f.fileClient.GetProperties(ctx, f.fileSystemName, f.path, action, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) } // UploadRange writes bytes to a file. // offset indiciates the offset at which to begin writing, in bytes. -func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeker) (*UpdatePathResponse, error) { +func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeker) (*PathUpdateResponse, error) { if offset < 0 { panic("offset must be >= 0") } @@ -128,21 +133,29 @@ func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeke if count == 0 { panic("body must contain readable data whose size is > 0") } - countAsStr := strconv.FormatInt(count, 10) - // TODO the go http client has a problem with PATCH and content-length header - // TODO we should investigate and report the issue + // TODO: the go http client has a problem with PATCH and content-length header + // we should investigate and report the issue + // Note: the "offending" code in the Go SDK is: func (t *transferWriter) shouldSendContentLength() bool + // That code suggests that a workaround would be to specify a Transfer-Encoding of "identity", + // but we haven't yet found any way to actually set that header, so that workaround does't + // seem to work. (Just setting Transfer-Encoding like a normal header doesn't seem to work.) + // Looks like it might actually be impossible to set + // the Transfer-Encoding header, because bradfitz wrote: "as a general rule of thumb, you don't get to mess + // with [that field] too much. The net/http package owns much of its behavior." + // https://grokbase.com/t/gg/golang-nuts/15bg66ryd9/go-nuts-cant-write-encoding-other-than-chunked-in-the-transfer-encoding-field-of-http-request overrideHttpVerb := "PATCH" // TransactionalContentMD5 isn't supported currently. - return f.fileClient.UpdatePath(ctx, "append", f.fileSystemName, f.path, &offset, - nil, &countAsStr, nil, nil, nil, nil, + return f.fileClient.Update(ctx, PathUpdateActionAppend, f.fileSystemName, f.path, &offset, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, &overrideHttpVerb, body, nil, nil, nil) + nil, nil, nil, nil, nil, nil, &overrideHttpVerb, body, nil, nil, nil) } // flushes writes previously uploaded data to a file -func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*UpdatePathResponse, error) { +// The contentMd5 parameter, if not nil, should represent the MD5 hash that has been computed for the file as whole +func (f FileURL) FlushData(ctx context.Context, fileSize int64, contentMd5 []byte) (*PathUpdateResponse, error) { if fileSize < 0 { panic("fileSize must be >= 0") } @@ -151,14 +164,24 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*UpdatePathResp // azcopy does not need this retainUncommittedData := false - // TODO the go http client has a problem with PATCH and content-length header - // TODO we should investigate and report the issue + var md5InBase64 *string = nil + if len(contentMd5) > 0 { + enc := base64.StdEncoding.EncodeToString(contentMd5) + md5InBase64 = &enc + } + + // TODO: the go http client has a problem with PATCH and content-length header + // we should investigate and report the issue + // See similar todo, with larger comments, in AppendData overrideHttpVerb := "PATCH" + // TODO: feb 2019 API update: review the use of closeParameter here. Should it be true? + // Doc implies only make it true if this is the end of the file + // TransactionalContentMD5 isn't supported currently. - return f.fileClient.UpdatePath(ctx, "flush", f.fileSystemName, f.path, &fileSize, + return f.fileClient.Update(ctx, PathUpdateActionFlush, f.fileSystemName, f.path, &fileSize, &retainUncommittedData, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, + nil, nil, nil, md5InBase64, nil, nil, nil, nil, nil, nil, nil, nil, - &overrideHttpVerb, nil, nil, nil, nil) + nil, &overrideHttpVerb, nil, nil, nil, nil) } diff --git a/azbfs/url_filesystem.go b/azbfs/url_filesystem.go index 929978bbe..eb3d87e9a 100644 --- a/azbfs/url_filesystem.go +++ b/azbfs/url_filesystem.go @@ -7,11 +7,9 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" ) -const fileSystemResourceName = "filesystem" // constant value for the resource query parameter - // A FileSystemURL represents a URL to the Azure Storage Blob File System allowing you to manipulate its directories and files. type FileSystemURL struct { - fileSystemClient managementClient + fileSystemClient filesystemClient name string } @@ -20,7 +18,7 @@ func NewFileSystemURL(url url.URL, p pipeline.Pipeline) FileSystemURL { if p == nil { panic("p can't be nil") } - fileSystemClient := newManagementClient(url, p) + fileSystemClient := newFilesystemClient(url, p) urlParts := NewBfsURLParts(url) return FileSystemURL{fileSystemClient: fileSystemClient, name: urlParts.FileSystemName} @@ -62,17 +60,17 @@ func (s FileSystemURL) NewRootDirectoryURL() DirectoryURL { // Create creates a new file system within a storage account. If a file system with the same name already exists, the operation fails. // quotaInGB specifies the maximum size of the file system in gigabytes, 0 means you accept service's default quota. -func (s FileSystemURL) Create(ctx context.Context) (*CreateFilesystemResponse, error) { - return s.fileSystemClient.CreateFilesystem(ctx, s.name, fileSystemResourceName, nil, nil, nil, nil) +func (s FileSystemURL) Create(ctx context.Context) (*FilesystemCreateResponse, error) { + return s.fileSystemClient.Create(ctx, s.name, nil, nil, nil, nil) } // Delete marks the specified file system for deletion. // The file system and any files contained within it are later deleted during garbage collection. -func (s FileSystemURL) Delete(ctx context.Context) (*DeleteFilesystemResponse, error) { - return s.fileSystemClient.DeleteFilesystem(ctx, s.name, fileSystemResourceName, nil, nil, nil, nil, nil) +func (s FileSystemURL) Delete(ctx context.Context) (*FilesystemDeleteResponse, error) { + return s.fileSystemClient.Delete(ctx, s.name, nil, nil, nil, nil, nil) } // GetProperties returns all user-defined metadata and system properties for the specified file system or file system snapshot. -func (s FileSystemURL) GetProperties(ctx context.Context) (*GetFilesystemPropertiesResponse, error) { - return s.fileSystemClient.GetFilesystemProperties(ctx, s.name, fileSystemResourceName, nil, nil, nil) +func (s FileSystemURL) GetProperties(ctx context.Context) (*FilesystemGetPropertiesResponse, error) { + return s.fileSystemClient.GetProperties(ctx, s.name, nil, nil, nil) } diff --git a/azbfs/zc_retry_reader.go b/azbfs/zc_retry_reader.go index 988566745..05cb79b64 100644 --- a/azbfs/zc_retry_reader.go +++ b/azbfs/zc_retry_reader.go @@ -5,6 +5,8 @@ import ( "io" "net" "net/http" + "strings" + "sync" ) // HTTPGetter is a function type that refers to a method that performs an HTTP GET operation. @@ -26,6 +28,9 @@ type HTTPGetterInfo struct { ETag string } +// FailedReadNotifier is a function type that represents the notification function called when a read fails +type FailedReadNotifier func(failureCount int, lastError error, offset int64, count int64, willRetry bool) + // RetryReaderOptions contains properties which can help to decide when to do retry. type RetryReaderOptions struct { // MaxRetryRequests specifies the maximum number of HTTP GET requests that will be made @@ -34,6 +39,20 @@ type RetryReaderOptions struct { MaxRetryRequests int doInjectError bool doInjectErrorRound int + + // NotifyFailedRead is called, if non-nil, after any failure to read. Expected usage is diagnostic logging. + NotifyFailedRead FailedReadNotifier + + // TreatEarlyCloseAsError can be set to true to prevent retries after "read on closed response body". By default, + // retryReader has the following special behaviour: closing the response body before it is all read is treated as a + // retryable error. This is to allow callers to force a retry by closing the body from another goroutine (e.g. if the = + // read is too slow, caller may want to force a retry in the hope that the retry will be quicker). If + // TreatEarlyCloseAsError is true, then retryReader's special behaviour is suppressed, and "read on closed body" is instead + // treated as a fatal (non-retryable) error. + // Note that setting TreatEarlyCloseAsError only guarantees that Closing will produce a fatal error if the Close happens + // from the same "thread" (goroutine) as Read. Concurrent Close calls from other goroutines may instead produce network errors + // which will be retried. + TreatEarlyCloseAsError bool } // retryReader implements io.ReaderCloser methods. @@ -43,11 +62,14 @@ type RetryReaderOptions struct { // through reading from the new response. type retryReader struct { ctx context.Context - response *http.Response info HTTPGetterInfo countWasBounded bool o RetryReaderOptions getter HTTPGetter + + // we support Close-ing during Reads (from other goroutines), so we protect the shared state, which is response + responseMu *sync.Mutex + response *http.Response } // NewRetryReader creates a retry reader. @@ -62,7 +84,20 @@ func NewRetryReader(ctx context.Context, initialResponse *http.Response, if o.MaxRetryRequests < 0 { panic("o.MaxRetryRequests must be >= 0") } - return &retryReader{ctx: ctx, getter: getter, info: info, countWasBounded: info.Count != CountToEnd, response: initialResponse, o: o} + return &retryReader{ + ctx: ctx, + getter: getter, + info: info, + countWasBounded: info.Count != CountToEnd, + response: initialResponse, + responseMu: &sync.Mutex{}, + o: o} +} + +func (s *retryReader) setResponse(r *http.Response) { + s.responseMu.Lock() + defer s.responseMu.Unlock() + s.response = r } func (s *retryReader) Read(p []byte) (n int, err error) { @@ -73,15 +108,19 @@ func (s *retryReader) Read(p []byte) (n int, err error) { return 0, io.EOF } - if s.response == nil { // We don't have a response stream to read from, try to get one. - response, err := s.getter(s.ctx, s.info) + s.responseMu.Lock() + resp := s.response + s.responseMu.Unlock() + if resp == nil { // We don't have a response stream to read from, try to get one. + newResponse, err := s.getter(s.ctx, s.info) if err != nil { return 0, err } // Successful GET; this is the network stream we'll read from. - s.response = response + s.setResponse(newResponse) + resp = newResponse } - n, err := s.response.Body.Read(p) // Read from the stream + n, err := resp.Body.Read(p) // Read from the stream (this will return non-nil err if forceRetry is called, from another goroutine, while it is running) // Injection mechanism for testing. if s.o.doInjectError && try == s.o.doInjectErrorRound { @@ -96,23 +135,49 @@ func (s *retryReader) Read(p []byte) (n int, err error) { } return n, err // Return the return to the caller } - s.Close() // Error, close stream - s.response = nil // Our stream is no longer good + s.Close() // Error, close stream + s.setResponse(nil) // Our stream is no longer good // Check the retry count and error code, and decide whether to retry. - if try >= s.o.MaxRetryRequests { - return n, err // All retries exhausted + retriesExhausted := try >= s.o.MaxRetryRequests + _, isNetError := err.(net.Error) + willRetry := (isNetError || s.wasRetryableEarlyClose(err)) && !retriesExhausted + + // Notify, for logging purposes, of any failures + if s.o.NotifyFailedRead != nil { + failureCount := try + 1 // because try is zero-based + s.o.NotifyFailedRead(failureCount, err, s.info.Offset, s.info.Count, willRetry) } - if _, ok := err.(net.Error); ok { + if willRetry { continue // Loop around and try to get and read from new stream. } - return n, err // Not retryable, just return + return n, err // Not retryable, or retries exhausted, so just return } } +// By default, we allow early Closing, from another concurrent goroutine, to be used to force a retry +// Is this safe, to close early from another goroutine? Early close ultimately ends up calling +// net.Conn.Close, and that is documented as "Any blocked Read or Write operations will be unblocked and return errors" +// which is exactly the behaviour we want. +// NOTE: that if caller has forced an early Close from a separate goroutine (separate from the Read) +// then there are two different types of error that may happen - either the one one we check for here, +// or a net.Error (due to closure of connection). Which one happens depends on timing. We only need this routine +// to check for one, since the other is a net.Error, which our main Read retry loop is already handing. +func (s *retryReader) wasRetryableEarlyClose(err error) bool { + if s.o.TreatEarlyCloseAsError { + return false // user wants all early closes to be errors, and so not retryable + } + // unfortunately, http.errReadOnClosedResBody is private, so the best we can do here is to check for its text + return strings.HasSuffix(err.Error(), ReadOnClosedBodyMessage) +} + +const ReadOnClosedBodyMessage = "read on closed response body" + func (s *retryReader) Close() error { + s.responseMu.Lock() + defer s.responseMu.Unlock() if s.response != nil && s.response.Body != nil { return s.response.Body.Close() } diff --git a/azbfs/zt_retry_reader_test.go b/azbfs/zt_retry_reader_test.go new file mode 100644 index 000000000..cc5b16321 --- /dev/null +++ b/azbfs/zt_retry_reader_test.go @@ -0,0 +1,330 @@ +package azbfs_test + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "github.com/Azure/azure-storage-azcopy/azbfs" + "io" + "net" + "net/http" + "time" + + chk "gopkg.in/check.v1" +) + +// Testings for RetryReader +// This reader return one byte through each Read call +type perByteReader struct { + RandomBytes []byte // Random generated bytes + + byteCount int // Bytes can be returned before EOF + currentByteIndex int // Bytes that have already been returned. + doInjectError bool + doInjectErrorByteIndex int + doInjectTimes int + injectedError error + + // sleepDuraion and closeChannel are only use in "forced cancellation" tests + sleepDuration time.Duration + closeChannel chan struct{} +} + +func newPerByteReader(byteCount int) *perByteReader { + perByteReader := perByteReader{ + byteCount: byteCount, + closeChannel: nil, + } + + perByteReader.RandomBytes = make([]byte, byteCount) + _, _ = rand.Read(perByteReader.RandomBytes) + + return &perByteReader +} + +func newSingleUsePerByteReader(contents []byte) *perByteReader { + perByteReader := perByteReader{ + byteCount: len(contents), + closeChannel: make(chan struct{}, 10), + } + + perByteReader.RandomBytes = contents + + return &perByteReader +} + +func (r *perByteReader) Read(b []byte) (n int, err error) { + if r.doInjectError && r.doInjectErrorByteIndex == r.currentByteIndex && r.doInjectTimes > 0 { + r.doInjectTimes-- + return 0, r.injectedError + } + + if r.currentByteIndex < r.byteCount { + n = copy(b, r.RandomBytes[r.currentByteIndex:r.currentByteIndex+1]) + r.currentByteIndex += n + + // simulate a delay, which may be successful or, if we're closed from another go-routine, may return an + // error + select { + case <-r.closeChannel: + return n, errors.New(azbfs.ReadOnClosedBodyMessage) + case <-time.After(r.sleepDuration): + return n, nil + } + } + + return 0, io.EOF +} + +func (r *perByteReader) Close() error { + if r.closeChannel != nil { + r.closeChannel <- struct{}{} + } + return nil +} + +// Test normal retry succeed, note initial response not provided. +// Tests both with and without notification of failures +func (r *aztestsSuite) TestRetryReaderReadWithRetry(c *chk.C) { + // Test twice, the second time using the optional "logging"/notification callback for failed tries + // We must test both with and without the callback, since be testing without + // we are testing that it is, indeed, optional to provide the callback + for _, logThisRun := range []bool{false, true} { + + // Extra setup for testing notification of failures (i.e. of unsuccessful tries) + failureMethodNumCalls := 0 + failureWillRetryCount := 0 + failureLastReportedFailureCount := -1 + var failureLastReportedError error = nil + failureMethod := func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + failureMethodNumCalls++ + if willRetry { + failureWillRetryCount++ + } + failureLastReportedFailureCount = failureCount + failureLastReportedError = lastError + } + + // Main test setup + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 0 + body.doInjectTimes = 1 + body.injectedError = &net.DNSError{IsTemporary: true} + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + httpGetterInfo := azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)} + initResponse, err := getter(context.Background(), httpGetterInfo) + c.Assert(err, chk.IsNil) + + rrOptions := azbfs.RetryReaderOptions{MaxRetryRequests: 1} + if logThisRun { + rrOptions.NotifyFailedRead = failureMethod + } + retryReader := azbfs.NewRetryReader(context.Background(), initResponse, httpGetterInfo, rrOptions, getter) + + // should fail and succeed through retry + can := make([]byte, 1) + n, err := retryReader.Read(can) + c.Assert(n, chk.Equals, 1) + c.Assert(err, chk.IsNil) + + // check "logging", if it was enabled + if logThisRun { + // We only expect one failed try in this test + // And the notification method is not called for successes + c.Assert(failureMethodNumCalls, chk.Equals, 1) // this is the number of calls we counted + c.Assert(failureWillRetryCount, chk.Equals, 1) // the sole failure was retried + c.Assert(failureLastReportedFailureCount, chk.Equals, 1) // this is the number of failures reported by the notification method + c.Assert(failureLastReportedError, chk.NotNil) + } + // should return EOF + n, err = retryReader.Read(can) + c.Assert(n, chk.Equals, 0) + c.Assert(err, chk.Equals, io.EOF) + } +} + +// Test normal retry fail as retry Count not enough. +func (r *aztestsSuite) TestRetryReaderReadNegativeNormalFail(c *chk.C) { + // Extra setup for testing notification of failures (i.e. of unsuccessful tries) + failureMethodNumCalls := 0 + failureWillRetryCount := 0 + failureLastReportedFailureCount := -1 + var failureLastReportedError error = nil + failureMethod := func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + failureMethodNumCalls++ + if willRetry { + failureWillRetryCount++ + } + failureLastReportedFailureCount = failureCount + failureLastReportedError = lastError + } + + // Main test setup + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 0 + body.doInjectTimes = 2 + body.injectedError = &net.DNSError{IsTemporary: true} + + startResponse := http.Response{} + startResponse.Body = body + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + rrOptions := azbfs.RetryReaderOptions{ + MaxRetryRequests: 1, + NotifyFailedRead: failureMethod} + retryReader := azbfs.NewRetryReader(context.Background(), &startResponse, azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)}, rrOptions, getter) + + // should fail + can := make([]byte, 1) + n, err := retryReader.Read(can) + c.Assert(n, chk.Equals, 0) + c.Assert(err, chk.Equals, body.injectedError) + + // Check that we recieved the right notification callbacks + // We only expect two failed tries in this test, but only one + // of the would have had willRetry = true + c.Assert(failureMethodNumCalls, chk.Equals, 2) // this is the number of calls we counted + c.Assert(failureWillRetryCount, chk.Equals, 1) // only the first failure was retried + c.Assert(failureLastReportedFailureCount, chk.Equals, 2) // this is the number of failures reported by the notification method + c.Assert(failureLastReportedError, chk.NotNil) +} + +// Test boundary case when Count equals to 0 and fail. +func (r *aztestsSuite) TestRetryReaderReadCount0(c *chk.C) { + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 1 + body.doInjectTimes = 1 + body.injectedError = &net.DNSError{IsTemporary: true} + + startResponse := http.Response{} + startResponse.Body = body + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + retryReader := azbfs.NewRetryReader(context.Background(), &startResponse, azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)}, azbfs.RetryReaderOptions{MaxRetryRequests: 1}, getter) + + // should consume the only byte + can := make([]byte, 1) + n, err := retryReader.Read(can) + c.Assert(n, chk.Equals, 1) + c.Assert(err, chk.IsNil) + + // should not read when Count=0, and should return EOF + n, err = retryReader.Read(can) + c.Assert(n, chk.Equals, 0) + c.Assert(err, chk.Equals, io.EOF) +} + +func (r *aztestsSuite) TestRetryReaderReadNegativeNonRetriableError(c *chk.C) { + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 0 + body.doInjectTimes = 1 + body.injectedError = fmt.Errorf("not retriable error") + + startResponse := http.Response{} + startResponse.Body = body + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + retryReader := azbfs.NewRetryReader(context.Background(), &startResponse, azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)}, azbfs.RetryReaderOptions{MaxRetryRequests: 2}, getter) + + dest := make([]byte, 1) + _, err := retryReader.Read(dest) + c.Assert(err, chk.Equals, body.injectedError) +} + +// Test the case where we programmatically force a retry to happen, via closing the body early from another goroutine +// Unlike the retries orchestrated elsewhere in this test file, which simulate network failures for the +// purposes of unit testing, here we are testing the cancellation mechanism that is exposed to +// consumers of the API, to allow programmatic forcing of retries (e.g. if the consumer deems +// the read to be taking too long, they may force a retry in the hope of better performance next time). +func (r *aztestsSuite) TestRetryReaderReadWithForcedRetry(c *chk.C) { + + for _, enableRetryOnEarlyClose := range []bool{false, true} { + + // use the notification callback, so we know that the retry really did happen + failureMethodNumCalls := 0 + failureMethod := func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + failureMethodNumCalls++ + } + + // Main test setup + byteCount := 10 // so multiple passes through read loop will be required + sleepDuration := 100 * time.Millisecond + randBytes := make([]byte, byteCount) + _, _ = rand.Read(randBytes) + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + body := newSingleUsePerByteReader(randBytes) // make new one every time, since we force closes in this test, and its unusable after a close + body.sleepDuration = sleepDuration + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + httpGetterInfo := azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)} + initResponse, err := getter(context.Background(), httpGetterInfo) + c.Assert(err, chk.IsNil) + + rrOptions := azbfs.RetryReaderOptions{MaxRetryRequests: 2, TreatEarlyCloseAsError: !enableRetryOnEarlyClose} + rrOptions.NotifyFailedRead = failureMethod + retryReader := azbfs.NewRetryReader(context.Background(), initResponse, httpGetterInfo, rrOptions, getter) + + // set up timed cancellation from separate goroutine + go func() { + time.Sleep(sleepDuration * 5) + retryReader.Close() + }() + + // do the read (should fail, due to forced cancellation, and succeed through retry) + output := make([]byte, byteCount) + n, err := io.ReadFull(retryReader, output) + if enableRetryOnEarlyClose { + c.Assert(n, chk.Equals, byteCount) + c.Assert(err, chk.IsNil) + c.Assert(output, chk.DeepEquals, randBytes) + c.Assert(failureMethodNumCalls, chk.Equals, 1) // assert that the cancellation did indeed happen + } else { + c.Assert(err, chk.NotNil) + } + } +} + +// End testings for RetryReader diff --git a/azbfs/zt_url_file_test.go b/azbfs/zt_url_file_test.go index 6e0d7c66e..20ac3580c 100644 --- a/azbfs/zt_url_file_test.go +++ b/azbfs/zt_url_file_test.go @@ -193,7 +193,7 @@ func (s *FileURLSuite) TestUploadDownloadRoundTrip(c *chk.C) { c.Assert(pResp.Date(), chk.Not(chk.Equals), "") // Flush data - fResp, err := fileURL.FlushData(context.Background(), 4096) + fResp, err := fileURL.FlushData(context.Background(), 4096, make([]byte, 0)) c.Assert(err, chk.IsNil) c.Assert(fResp.StatusCode(), chk.Equals, http.StatusOK) c.Assert(fResp.ETag(), chk.Not(chk.Equals), "") @@ -206,7 +206,7 @@ func (s *FileURLSuite) TestUploadDownloadRoundTrip(c *chk.C) { resp, err := fileURL.Download(context.Background(), 0, 1024) c.Assert(err, chk.IsNil) c.Assert(resp.StatusCode(), chk.Equals, http.StatusPartialContent) - c.Assert(resp.ContentLength(), chk.Equals, "1024") + c.Assert(resp.ContentLength(), chk.Equals, int64(1024)) c.Assert(resp.ContentType(), chk.Equals, "application/octet-stream") c.Assert(resp.Status(), chk.Not(chk.Equals), "") @@ -219,7 +219,7 @@ func (s *FileURLSuite) TestUploadDownloadRoundTrip(c *chk.C) { resp, err = fileURL.Download(context.Background(), 0, 0) c.Assert(err, chk.IsNil) c.Assert(resp.StatusCode(), chk.Equals, http.StatusOK) - c.Assert(resp.ContentLength(), chk.Equals, "4096") + c.Assert(resp.ContentLength(), chk.Equals, int64(4096)) c.Assert(resp.Date(), chk.Not(chk.Equals), "") c.Assert(resp.ETag(), chk.Not(chk.Equals), "") c.Assert(resp.LastModified(), chk.Not(chk.Equals), "") diff --git a/azbfs/zz_generated_client.go b/azbfs/zz_generated_client.go index 3cfcb281c..1288425f2 100644 --- a/azbfs/zz_generated_client.go +++ b/azbfs/zz_generated_client.go @@ -4,20 +4,13 @@ package azbfs // Changes may cause incorrect behavior and will be lost if the code is regenerated. import ( - "context" - "encoding/json" "github.com/Azure/azure-pipeline-go/pipeline" - "io" - "io/ioutil" - "net/http" "net/url" - "strconv" ) const ( // ServiceVersion specifies the version of the operations used in this package. - ServiceVersion = "2018-03-28" - //ServiceVersion = "2018-06-17" //TODO uncomment when service is ready + ServiceVersion = "2018-11-09" ) // managementClient is the base client for Azbfs. @@ -43,1270 +36,3 @@ func (mc managementClient) URL() url.URL { func (mc managementClient) Pipeline() pipeline.Pipeline { return mc.p } - -// CreateFilesystem create a filesystem rooted at the specified location. If the filesystem already exists, the -// operation fails. This operation does not support conditional HTTP requests. -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. xMsProperties is user-defined properties to be stored with the filesystem, in the format of a -// comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is base64 encoded. -// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) CreateFilesystem(ctx context.Context, filesystem string, resource string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*CreateFilesystemResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.createFilesystemPreparer(filesystem, resource, xMsProperties, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createFilesystemResponder}, req) - if err != nil { - return nil, err - } - return resp.(*CreateFilesystemResponse), err -} - -// createFilesystemPreparer prepares the CreateFilesystem request. -func (client managementClient) createFilesystemPreparer(filesystem string, resource string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PUT", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// createFilesystemResponder handles the response to the CreateFilesystem request. -func (client managementClient) createFilesystemResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusCreated) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &CreateFilesystemResponse{rawResponse: resp.Response()}, err -} - -// CreatePath create or rename a file or directory. By default, the destination is overwritten and if the -// destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. -// For more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// To fail if the destination already exists, use a conditional request with If-None-Match: "*". -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. resource is required only for -// Create File and Create Directory. The value must be "file" or "directory". continuation is optional. When renaming -// a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be -// renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is -// returned in the response, it must be specified in a subsequent invocation of the rename operation to continue -// renaming the directory. mode is optional. Valid only when namespace is enabled. This parameter determines the -// behavior of the rename operation. The value must be "legacy" or "posix", and the default value will be "posix". -// cacheControl is optional. The service stores this value and includes it in the "Cache-Control" response header for -// "Read File" operations for "Read File" operations. contentEncoding is optional. Specifies which content encodings -// have been applied to the file. This value is returned to the client when the "Read File" operation is performed. -// contentLanguage is optional. Specifies the natural language used by the intended audience for the file. -// contentDisposition is optional. The service stores this value and includes it in the "Content-Disposition" response -// header for "Read File" operations. xMsCacheControl is optional. The service stores this value and includes it in -// the "Cache-Control" response header for "Read File" operations. xMsContentType is optional. The service stores this -// value and includes it in the "Content-Type" response header for "Read File" operations. xMsContentEncoding is -// optional. The service stores this value and includes it in the "Content-Encoding" response header for "Read File" -// operations. xMsContentLanguage is optional. The service stores this value and includes it in the "Content-Language" -// response header for "Read File" operations. xMsContentDisposition is optional. The service stores this value and -// includes it in the "Content-Disposition" response header for "Read File" operations. xMsRenameSource is an optional -// file or directory to be renamed. The value must have the following format: "/{filesysystem}/{path}". If -// "x-ms-properties" is specified, the properties will overwrite the existing properties; otherwise, the existing -// properties will be preserved. xMsLeaseID is optional. A lease ID for the path specified in the URI. The path to be -// overwritten must have an active lease and the lease ID must match. xMsProposedLeaseID is optional for create -// operations. Required when "x-ms-lease-action" is used. A lease will be acquired using the proposed ID when the -// resource is created. xMsSourceLeaseID is optional for rename operations. A lease ID for the source path. The -// source path must have an active lease and the lease ID must match. xMsProperties is optional. User-defined -// properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs -// "n1=v1, n2=v2, ...", where each value is base64 encoded. xMsPermissions is optional and only valid if Hierarchical -// Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and -// others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both -// symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. ifMatch is optional. An ETag value. -// Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must -// be specified in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this -// header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be -// specified in quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform the -// operation only if the resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A -// date and time value. Specify this header to perform the operation only if the resource has not been modified since -// the specified date and time. xMsSourceIfMatch is optional. An ETag value. Specify this header to perform the rename -// operation only if the source's ETag matches the value specified. The ETag must be specified in quotes. -// xMsSourceIfNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to perform -// the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in -// quotes. xMsSourceIfModifiedSince is optional. A date and time value. Specify this header to perform the rename -// operation only if the source has been modified since the specified date and time. xMsSourceIfUnmodifiedSince is -// optional. A date and time value. Specify this header to perform the rename operation only if the source has not been -// modified since the specified date and time. xMsClientRequestID is a UUID recorded in the analytics logs for -// troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The period begins when -// the request is received by the service. If the timeout value elapses before the operation completes, the operation -// fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using -// shared key authorization. -func (client managementClient) CreatePath(ctx context.Context, filesystem string, pathParameter string, resource *string, continuation *string, mode *string, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsProposedLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*CreatePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: xMsProposedLeaseID, - constraints: []constraint{{target: "xMsProposedLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: xMsSourceLeaseID, - constraints: []constraint{{target: "xMsSourceLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsSourceLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.createPathPreparer(filesystem, pathParameter, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, xMsCacheControl, xMsContentType, xMsContentEncoding, xMsContentLanguage, xMsContentDisposition, xMsRenameSource, xMsLeaseID, xMsProposedLeaseID, xMsSourceLeaseID, xMsProperties, xMsPermissions, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsSourceIfMatch, xMsSourceIfNoneMatch, xMsSourceIfModifiedSince, xMsSourceIfUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createPathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*CreatePathResponse), err -} - -// createPathPreparer prepares the CreatePath request. -func (client managementClient) createPathPreparer(filesystem string, pathParameter string, resource *string, continuation *string, mode *string, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsProposedLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PUT", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if resource != nil && len(*resource) > 0 { - params.Set("resource", *resource) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if mode != nil && len(*mode) > 0 { - params.Set("mode", *mode) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if cacheControl != nil { - req.Header.Set("Cache-Control", *cacheControl) - } - if contentEncoding != nil { - req.Header.Set("Content-Encoding", *contentEncoding) - } - if contentLanguage != nil { - req.Header.Set("Content-Language", *contentLanguage) - } - if contentDisposition != nil { - req.Header.Set("Content-Disposition", *contentDisposition) - } - if xMsCacheControl != nil { - req.Header.Set("x-ms-cache-control", *xMsCacheControl) - } - if xMsContentType != nil { - req.Header.Set("x-ms-content-type", *xMsContentType) - } - if xMsContentEncoding != nil { - req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) - } - if xMsContentLanguage != nil { - req.Header.Set("x-ms-content-language", *xMsContentLanguage) - } - if xMsContentDisposition != nil { - req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) - } - if xMsRenameSource != nil { - req.Header.Set("x-ms-rename-source", *xMsRenameSource) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsProposedLeaseID != nil { - req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) - } - if xMsSourceLeaseID != nil { - req.Header.Set("x-ms-source-lease-id", *xMsSourceLeaseID) - } - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsPermissions != nil { - req.Header.Set("x-ms-permissions", *xMsPermissions) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsSourceIfMatch != nil { - req.Header.Set("x-ms-source-if-match", *xMsSourceIfMatch) - } - if xMsSourceIfNoneMatch != nil { - req.Header.Set("x-ms-source-if-none-match", *xMsSourceIfNoneMatch) - } - if xMsSourceIfModifiedSince != nil { - req.Header.Set("x-ms-source-if-modified-since", *xMsSourceIfModifiedSince) - } - if xMsSourceIfUnmodifiedSince != nil { - req.Header.Set("x-ms-source-if-unmodified-since", *xMsSourceIfUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// createPathResponder handles the response to the CreatePath request. -func (client managementClient) createPathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusCreated) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &CreatePathResponse{rawResponse: resp.Response()}, err -} - -// DeleteFilesystem marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same -// identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a -// filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional -// error information indicating that the filesystem is being deleted. All other operations, including operations on any -// files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being -// deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional -// Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if -// the resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time -// value. Specify this header to perform the operation only if the resource has not been modified since the specified -// date and time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. -// timeout is an optional operation timeout value in seconds. The period begins when the request is received by the -// service. If the timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the -// Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) DeleteFilesystem(ctx context.Context, filesystem string, resource string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*DeleteFilesystemResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.deleteFilesystemPreparer(filesystem, resource, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteFilesystemResponder}, req) - if err != nil { - return nil, err - } - return resp.(*DeleteFilesystemResponse), err -} - -// deleteFilesystemPreparer prepares the DeleteFilesystem request. -func (client managementClient) deleteFilesystemPreparer(filesystem string, resource string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("DELETE", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// deleteFilesystemResponder handles the response to the DeleteFilesystem request. -func (client managementClient) deleteFilesystemResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusAccepted) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &DeleteFilesystemResponse{rawResponse: resp.Response()}, err -} - -// DeletePath delete the file or directory. This operation supports conditional HTTP requests. For more information, -// see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. recursive is required and -// valid only when the resource is a directory. If "true", all paths beneath the directory will be deleted. If "false" -// and the directory is non-empty, an error occurs. continuation is optional. When deleting a directory, the number of -// paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a -// continuation token is returned in this response header. When a continuation token is returned in the response, it -// must be specified in a subsequent invocation of the delete operation to continue deleting the directory. xMsLeaseID -// is the lease ID must be specified if there is an active lease. ifMatch is optional. An ETag value. Specify this -// header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified -// in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to -// perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in -// quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the -// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. -// Specify this header to perform the operation only if the resource has not been modified since the specified date and -// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) DeletePath(ctx context.Context, filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*DeletePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.deletePathPreparer(filesystem, pathParameter, recursive, continuation, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deletePathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*DeletePathResponse), err -} - -// deletePathPreparer prepares the DeletePath request. -func (client managementClient) deletePathPreparer(filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("DELETE", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if recursive != nil { - params.Set("recursive", strconv.FormatBool(*recursive)) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// deletePathResponder handles the response to the DeletePath request. -func (client managementClient) deletePathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &DeletePathResponse{rawResponse: resp.Response()}, err -} - -// GetFilesystemProperties all system and user-defined filesystem properties are specified in the response headers. -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout -// is an optional operation timeout value in seconds. The period begins when the request is received by the service. If -// the timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) GetFilesystemProperties(ctx context.Context, filesystem string, resource string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*GetFilesystemPropertiesResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.getFilesystemPropertiesPreparer(filesystem, resource, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getFilesystemPropertiesResponder}, req) - if err != nil { - return nil, err - } - return resp.(*GetFilesystemPropertiesResponse), err -} - -// getFilesystemPropertiesPreparer prepares the GetFilesystemProperties request. -func (client managementClient) getFilesystemPropertiesPreparer(filesystem string, resource string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("HEAD", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// getFilesystemPropertiesResponder handles the response to the GetFilesystemProperties request. -func (client managementClient) getFilesystemPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &GetFilesystemPropertiesResponse{rawResponse: resp.Response()}, err -} - -// GetPathProperties get the properties for a file or directory, and optionally include the access control list. This -// operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob -// Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. action is optional. If the -// value is "getAccessControl" the access control list is returned in the response headers (Hierarchical Namespace must -// be enabled for the account). ifMatch is optional. An ETag value. Specify this header to perform the operation only -// if the resource's ETag matches the value specified. The ETag must be specified in quotes. ifNoneMatch is optional. -// An ETag value or the special wildcard ("*") value. Specify this header to perform the operation only if the -// resource's ETag does not match the value specified. The ETag must be specified in quotes. ifModifiedSince is -// optional. A date and time value. Specify this header to perform the operation only if the resource has been modified -// since the specified date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to -// perform the operation only if the resource has not been modified since the specified date and time. -// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) GetPathProperties(ctx context.Context, filesystem string, pathParameter string, action *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*GetPathPropertiesResponse, error) { - if err := validate([]validation{ - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.getPathPropertiesPreparer(filesystem, pathParameter, action, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPathPropertiesResponder}, req) - if err != nil { - return nil, err - } - return resp.(*GetPathPropertiesResponse), err -} - -// getPathPropertiesPreparer prepares the GetPathProperties request. -func (client managementClient) getPathPropertiesPreparer(filesystem string, pathParameter string, action *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("HEAD", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if action != nil && len(*action) > 0 { - params.Set("action", *action) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// getPathPropertiesResponder handles the response to the GetPathProperties request. -func (client managementClient) getPathPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &GetPathPropertiesResponse{rawResponse: resp.Response()}, err -} - -// LeasePath create and manage a lease to restrict write and delete access to the path. This operation supports -// conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// xMsLeaseAction is there are five lease actions: "acquire", "break", "change", "renew", and "release". Use "acquire" -// and specify the "x-ms-proposed-lease-id" and "x-ms-lease-duration" to acquire a new lease. Use "break" to break an -// existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease -// operation except break and release can be performed on the file. When a lease is successfully broken, the response -// indicates the interval in seconds until a new lease can be acquired. Use "change" and specify the current lease ID -// in "x-ms-lease-id" and the new lease ID in "x-ms-proposed-lease-id" to change the lease ID of an active lease. Use -// "renew" and specify the "x-ms-lease-id" to renew an existing lease. Use "release" and specify the "x-ms-lease-id" to -// release a lease. filesystem is the filesystem identifier. pathParameter is the file or directory path. -// xMsLeaseDuration is the lease duration is required to acquire a lease, and specifies the duration of the lease in -// seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. xMsLeaseBreakPeriod is the -// lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. -// The lease break duration must be between 0 and 60 seconds. xMsLeaseID is required when "x-ms-lease-action" is -// "renew", "change" or "release". For the renew and release actions, this must match the current lease ID. -// xMsProposedLeaseID is required when "x-ms-lease-action" is "acquire" or "change". A lease will be acquired with -// this lease ID if the operation is successful. ifMatch is optional. An ETag value. Specify this header to perform -// the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes. -// ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to perform the -// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. -// ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the -// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. -// Specify this header to perform the operation only if the resource has not been modified since the specified date and -// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) LeasePath(ctx context.Context, xMsLeaseAction string, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*LeasePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: xMsProposedLeaseID, - constraints: []constraint{{target: "xMsProposedLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.leasePathPreparer(xMsLeaseAction, filesystem, pathParameter, xMsLeaseDuration, xMsLeaseBreakPeriod, xMsLeaseID, xMsProposedLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.leasePathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*LeasePathResponse), err -} - -// leasePathPreparer prepares the LeasePath request. -func (client managementClient) leasePathPreparer(xMsLeaseAction string, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("POST", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - req.Header.Set("x-ms-lease-action", xMsLeaseAction) - if xMsLeaseDuration != nil { - req.Header.Set("x-ms-lease-duration", strconv.FormatInt(int64(*xMsLeaseDuration), 10)) - } - if xMsLeaseBreakPeriod != nil { - req.Header.Set("x-ms-lease-break-period", strconv.FormatInt(int64(*xMsLeaseBreakPeriod), 10)) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsProposedLeaseID != nil { - req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// leasePathResponder handles the response to the LeasePath request. -func (client managementClient) leasePathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &LeasePathResponse{rawResponse: resp.Response()}, err -} - -// ListFilesystems list filesystems and their properties in given account. -// -// resource is the value must be "account" for all account operations. prefix is filters results to filesystems within -// the specified prefix. continuation is the number of filesystems returned with each invocation is limited. If the -// number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header -// x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent -// invocation of the list operation to continue listing the filesystems. maxResults is an optional value that specifies -// the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 -// items. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is -// an optional operation timeout value in seconds. The period begins when the request is received by the service. If -// the timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) ListFilesystems(ctx context.Context, resource string, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ListFilesystemSchema, error) { - if err := validate([]validation{ - {targetValue: maxResults, - constraints: []constraint{{target: "maxResults", name: null, rule: false, - chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.listFilesystemsPreparer(resource, prefix, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listFilesystemsResponder}, req) - if err != nil { - return nil, err - } - return resp.(*ListFilesystemSchema), err -} - -// listFilesystemsPreparer prepares the ListFilesystems request. -func (client managementClient) listFilesystemsPreparer(resource string, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if prefix != nil && len(*prefix) > 0 { - params.Set("prefix", *prefix) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// listFilesystemsResponder handles the response to the ListFilesystems request. -func (client managementClient) listFilesystemsResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - result := &ListFilesystemSchema{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err := ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to read response body") - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil -} - -// ListPaths list filesystem paths and their properties. -// -// recursive is if "true", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If -// "directory" is specified, the list will only include paths that share the same root. filesystem is the filesystem -// identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the -// dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have -// between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem operations. directory is -// filters results to paths within the specified directory. An error occurs if the directory does not exist. -// continuation is the number of paths returned with each invocation is limited. If the number of paths to be returned -// exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation -// token is returned in the response, it must be specified in a subsequent invocation of the list operation to -// continue listing the paths. maxResults is an optional value that specifies the maximum number of items to return. If -// omitted or greater than 5,000, the response will include up to 5,000 items. xMsClientRequestID is a UUID recorded in -// the analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. -// The period begins when the request is received by the service. If the timeout value elapses before the operation -// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is -// required when using shared key authorization. -func (client managementClient) ListPaths(ctx context.Context, recursive bool, filesystem string, resource string, directory *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ListSchema, error) { - if err := validate([]validation{ - {targetValue: maxResults, - constraints: []constraint{{target: "maxResults", name: null, rule: false, - chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.listPathsPreparer(recursive, filesystem, resource, directory, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listPathsResponder}, req) - if err != nil { - return nil, err - } - return resp.(*ListSchema), err -} - -// listPathsPreparer prepares the ListPaths request. -func (client managementClient) listPathsPreparer(recursive bool, filesystem string, resource string, directory *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if directory != nil && len(*directory) > 0 { - params.Set("directory", *directory) - } - params.Set("recursive", strconv.FormatBool(recursive)) - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// listPathsResponder handles the response to the ListPaths request. -func (client managementClient) listPathsResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - result := &ListSchema{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err := ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to read response body") - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil -} - -// ReadPath read the contents of a file. For read operations, range requests are supported. This operation supports -// conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. rangeParameter is the HTTP -// Range request header specifies one or more byte ranges of the resource to be retrieved. ifMatch is optional. An -// ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. -// The ETag must be specified in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. -// Specify this header to perform the operation only if the resource's ETag does not match the value specified. The -// ETag must be specified in quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform -// the operation only if the resource has been modified since the specified date and time. ifUnmodifiedSince is -// optional. A date and time value. Specify this header to perform the operation only if the resource has not been -// modified since the specified date and time. xMsClientRequestID is a UUID recorded in the analytics logs for -// troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The period begins when -// the request is received by the service. If the timeout value elapses before the operation completes, the operation -// fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using -// shared key authorization. -func (client managementClient) ReadPath(ctx context.Context, filesystem string, pathParameter string, rangeParameter *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ReadPathResponse, error) { - if err := validate([]validation{ - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.readPathPreparer(filesystem, pathParameter, rangeParameter, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.readPathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*ReadPathResponse), err -} - -// readPathPreparer prepares the ReadPath request. -func (client managementClient) readPathPreparer(filesystem string, pathParameter string, rangeParameter *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if rangeParameter != nil { - req.Header.Set("Range", *rangeParameter) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// readPathResponder handles the response to the ReadPath request. -func (client managementClient) readPathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusPartialContent) - if resp == nil { - return nil, err - } - return &ReadPathResponse{rawResponse: resp.Response()}, err -} - -// SetFilesystemProperties set properties for the filesystem. This operation supports conditional HTTP requests. For -// more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. xMsProperties is optional. User-defined properties to be stored with the filesystem, in the format of a -// comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is base64 encoded. If the -// filesystem exists, any properties not included in the list will be removed. All properties are removed if the -// header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, -// then make a conditional request with the E-Tag and include values for all properties. ifModifiedSince is optional. A -// date and time value. Specify this header to perform the operation only if the resource has been modified since the -// specified date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the -// operation only if the resource has not been modified since the specified date and time. xMsClientRequestID is a UUID -// recorded in the analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value -// in seconds. The period begins when the request is received by the service. If the timeout value elapses before the -// operation completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. -// This is required when using shared key authorization. -func (client managementClient) SetFilesystemProperties(ctx context.Context, filesystem string, resource string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*SetFilesystemPropertiesResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.setFilesystemPropertiesPreparer(filesystem, resource, xMsProperties, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.setFilesystemPropertiesResponder}, req) - if err != nil { - return nil, err - } - return resp.(*SetFilesystemPropertiesResponse), err -} - -// setFilesystemPropertiesPreparer prepares the SetFilesystemProperties request. -func (client managementClient) setFilesystemPropertiesPreparer(filesystem string, resource string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PATCH", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// setFilesystemPropertiesResponder handles the response to the SetFilesystemProperties request. -func (client managementClient) setFilesystemPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &SetFilesystemPropertiesResponse{rawResponse: resp.Response()}, err -} - -// UpdatePath uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets -// properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a -// file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers -// for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// action is the action must be "append" to upload data to be appended to a file, "flush" to flush previously uploaded -// data to a file, "setProperties" to set the properties of a file or directory, or "setAccessControl" to set the -// owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be -// enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes -// permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are -// mutually exclusive. filesystem is the filesystem identifier. pathParameter is the file or directory path. position -// is this parameter allows the caller to upload data in parallel and control the order in which it is appended to the -// file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to -// the file. The value must be the position where the data is to be appended. Uploaded data is not immediately -// flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter -// must be specified and equal to the length of the file after all data has been written, and there must not be a -// request entity body included with the request. retainUncommittedData is valid only for flush operations. If "true", -// uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after -// the flush operation. The default is false. Data at offsets less than the specified position are written to the -// file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a -// future flush operation. contentLength is required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". -// Must be the length of the request content in bytes for "Append Data". xMsLeaseAction is optional. The lease action -// can be "renew" to renew an existing lease or "release" to release a lease. xMsLeaseID is the lease ID must be -// specified if there is an active lease. xMsCacheControl is optional and only valid for flush and set properties -// operations. The service stores this value and includes it in the "Cache-Control" response header for "Read File" -// operations. xMsContentType is optional and only valid for flush and set properties operations. The service stores -// this value and includes it in the "Content-Type" response header for "Read File" operations. xMsContentDisposition -// is optional and only valid for flush and set properties operations. The service stores this value and includes it -// in the "Content-Disposition" response header for "Read File" operations. xMsContentEncoding is optional and only -// valid for flush and set properties operations. The service stores this value and includes it in the -// "Content-Encoding" response header for "Read File" operations. xMsContentLanguage is optional and only valid for -// flush and set properties operations. The service stores this value and includes it in the "Content-Language" -// response header for "Read File" operations. xMsProperties is optional. User-defined properties to be stored with -// the file or directory, in the format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where -// each value is base64 encoded. Valid only for the setProperties operation. If the file or directory exists, any -// properties not included in the list will be removed. All properties are removed if the header is omitted. To merge -// new and existing properties, first get all existing properties and the current E-Tag, then make a conditional -// request with the E-Tag and include values for all properties. xMsOwner is optional and valid only for the -// setAccessControl operation. Sets the owner of the file or directory. xMsGroup is optional and valid only for the -// setAccessControl operation. Sets the owning group of the file or directory. xMsPermissions is optional and only -// valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the -// file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also -// supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction -// with x-ms-acl. xMsACL is optional and valid only for the setAccessControl operation. Sets POSIX access control -// rights on files and directories. The value is a comma-separated list of access control entries that fully replaces -// the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or -// group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". The scope must be "default" to -// indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the -// access ACL. There are four ACE types: "user" grants rights to the owner or a named user, "group" grants rights to -// the owning group or a named group, "mask" restricts rights granted to named users and the members of groups, and -// "other" grants rights to all users not found in any of the other entries. The user or group identifier is omitted -// for entries of type "mask" and "other". The user or group identifier is also omitted for the owner and owning -// group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the -// second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If -// access is not granted, the '-' character is used to denote that the permission is denied. For example, the following -// ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning -// group, and nothing to everyone else: "user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx". Invalid -// in conjunction with x-ms-permissions. ifMatch is optional for Flush Data and Set Properties, but invalid for Append -// Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value -// specified. The ETag must be specified in quotes. ifNoneMatch is optional for Flush Data and Set Properties, but -// invalid for Append Data. An ETag value or the special wildcard ("*") value. Specify this header to perform the -// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. -// ifModifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. -// Specify this header to perform the operation only if the resource has been modified since the specified date and -// time. ifUnmodifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time -// value. Specify this header to perform the operation only if the resource has not been modified since the specified -// date and time. xHTTPMethodOverride is optional. Override the http verb on the service side. Some older http clients -// do not support PATCH requestBody is valid only for append operations. The data to be uploaded and appended to the -// file. requestBody will be closed upon successful return. Callers should ensure closure when receiving an -// error.xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) UpdatePath(ctx context.Context, action string, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, contentLength *string, xMsLeaseAction *string, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*UpdatePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.updatePathPreparer(action, filesystem, pathParameter, position, retainUncommittedData, contentLength, xMsLeaseAction, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xHTTPMethodOverride, body, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.updatePathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*UpdatePathResponse), err -} - -// updatePathPreparer prepares the UpdatePath request. -func (client managementClient) updatePathPreparer(action string, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, contentLength *string, xMsLeaseAction *string, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PUT", client.url, body) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("action", action) - if position != nil { - params.Set("position", strconv.FormatInt(*position, 10)) - } - if retainUncommittedData != nil { - params.Set("retainUncommittedData", strconv.FormatBool(*retainUncommittedData)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if contentLength != nil { - req.Header.Set("Content-Length", *contentLength) - } - if xMsLeaseAction != nil { - req.Header.Set("x-ms-lease-action", *xMsLeaseAction) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsCacheControl != nil { - req.Header.Set("x-ms-cache-control", *xMsCacheControl) - } - if xMsContentType != nil { - req.Header.Set("x-ms-content-type", *xMsContentType) - } - if xMsContentDisposition != nil { - req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) - } - if xMsContentEncoding != nil { - req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) - } - if xMsContentLanguage != nil { - req.Header.Set("x-ms-content-language", *xMsContentLanguage) - } - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsOwner != nil { - req.Header.Set("x-ms-owner", *xMsOwner) - } - if xMsGroup != nil { - req.Header.Set("x-ms-group", *xMsGroup) - } - if xMsPermissions != nil { - req.Header.Set("x-ms-permissions", *xMsPermissions) - } - if xMsACL != nil { - req.Header.Set("x-ms-acl", *xMsACL) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xHTTPMethodOverride != nil { - req.Header.Set("x-http-method-override", *xHTTPMethodOverride) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// updatePathResponder handles the response to the UpdatePath request. -func (client managementClient) updatePathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusAccepted) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &UpdatePathResponse{rawResponse: resp.Response()}, err -} diff --git a/azbfs/zz_generated_filesystem.go b/azbfs/zz_generated_filesystem.go new file mode 100644 index 000000000..49aab19ad --- /dev/null +++ b/azbfs/zz_generated_filesystem.go @@ -0,0 +1,537 @@ +package azbfs + +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + // begin manual edit to generated code + "context" + "encoding/json" + "github.com/Azure/azure-pipeline-go/pipeline" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + // end manual edit +) + +// filesystemClient is the azure Data Lake Storage provides storage for Hadoop and other big data workloads. +type filesystemClient struct { + managementClient +} + +// newFilesystemClient creates an instance of the filesystemClient client. +func newFilesystemClient(url url.URL, p pipeline.Pipeline) filesystemClient { + return filesystemClient{newManagementClient(url, p)} +} + +// Create create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. +// This operation does not support conditional HTTP requests. +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. xMsProperties is user-defined properties to be stored with the +// filesystem, in the format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is +// a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. +// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client filesystemClient) Create(ctx context.Context, filesystem string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemCreateResponse, error) { + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.createPreparer(filesystem, xMsProperties, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemCreateResponse), err +} + +// createPreparer prepares the Create request. +func (client filesystemClient) createPreparer(filesystem string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PUT", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// createResponder handles the response to the Create request. +func (client filesystemClient) createResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK, http.StatusCreated) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemCreateResponse{rawResponse: resp.Response()}, err +} + +// Delete marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier +// cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem +// with the same identifier will fail with status code 409 (Conflict), with the service returning additional error +// information indicating that the filesystem is being deleted. All other operations, including operations on any files +// or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being +// deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional +// Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. ifModifiedSince is optional. A date and time value. Specify this +// header to perform the operation only if the resource has been modified since the specified date and time. +// ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has not been modified since the specified date and time. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client filesystemClient) Delete(ctx context.Context, filesystem string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemDeleteResponse, error) { + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.deletePreparer(filesystem, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemDeleteResponse), err +} + +// deletePreparer prepares the Delete request. +func (client filesystemClient) deletePreparer(filesystem string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("DELETE", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// deleteResponder handles the response to the Delete request. +func (client filesystemClient) deleteResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK, http.StatusAccepted) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemDeleteResponse{rawResponse: resp.Response()}, err +} + +// GetProperties all system and user-defined filesystem properties are specified in the response headers. +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. xMsClientRequestID is a UUID recorded in the analytics logs for +// troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The period begins when +// the request is received by the service. If the timeout value elapses before the operation completes, the operation +// fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using +// shared key authorization. +func (client filesystemClient) GetProperties(ctx context.Context, filesystem string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemGetPropertiesResponse, error) { + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.getPropertiesPreparer(filesystem, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPropertiesResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemGetPropertiesResponse), err +} + +// getPropertiesPreparer prepares the GetProperties request. +func (client filesystemClient) getPropertiesPreparer(filesystem string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("HEAD", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// getPropertiesResponder handles the response to the GetProperties request. +func (client filesystemClient) getPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemGetPropertiesResponse{rawResponse: resp.Response()}, err +} + +// List list filesystems and their properties in given account. +// +// prefix is filters results to filesystems within the specified prefix. continuation is the number of filesystems +// returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a +// continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in +// the response, it must be specified in a subsequent invocation of the list operation to continue listing the +// filesystems. maxResults is an optional value that specifies the maximum number of items to return. If omitted or +// greater than 5,000, the response will include up to 5,000 items. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client filesystemClient) List(ctx context.Context, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemList, error) { + if err := validate([]validation{ + {targetValue: maxResults, + constraints: []constraint{{target: "maxResults", name: null, rule: false, + chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.listPreparer(prefix, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemList), err +} + +// listPreparer prepares the List request. +func (client filesystemClient) listPreparer(prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "account") + if prefix != nil && len(*prefix) > 0 { + params.Set("prefix", *prefix) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// listResponder handles the response to the List request. +func (client filesystemClient) listResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + result := &FilesystemList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err := ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil +} + +// ListPaths list filesystem paths and their properties. +// +// recursive is if "true", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If +// "directory" is specified, the list will only include paths that share the same root. filesystem is the filesystem +// identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the +// dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have +// between 3 and 63 characters. directory is filters results to paths within the specified directory. An error occurs +// if the directory does not exist. continuation is the number of paths returned with each invocation is limited. If +// the number of paths to be returned exceeds this limit, a continuation token is returned in the response header +// x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent +// invocation of the list operation to continue listing the paths. maxResults is an optional value that specifies the +// maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items. +// upn is optional. Valid only when Hierarchical Namespace is enabled for the account. If "true", the user identity +// values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory +// Object IDs to User Principal Names. If "false", the values will be returned as Azure Active Directory Object IDs. +// The default value is false. Note that group and application Object IDs are not translated because they do not have +// unique friendly names. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and +// correlation. timeout is an optional operation timeout value in seconds. The period begins when the request is +// received by the service. If the timeout value elapses before the operation completes, the operation fails. xMsDate +// is specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key +// authorization. +func (client filesystemClient) ListPaths(ctx context.Context, recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathList, error) { + if err := validate([]validation{ + {targetValue: maxResults, + constraints: []constraint{{target: "maxResults", name: null, rule: false, + chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.listPathsPreparer(recursive, filesystem, directory, continuation, maxResults, upn, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listPathsResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathList), err +} + +// listPathsPreparer prepares the ListPaths request. +func (client filesystemClient) listPathsPreparer(recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if directory != nil && len(*directory) > 0 { + params.Set("directory", *directory) + } + params.Set("recursive", strconv.FormatBool(recursive)) + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// listPathsResponder handles the response to the ListPaths request. +func (client filesystemClient) listPathsResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + result := &PathList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err := ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil +} + +// SetProperties set properties for the filesystem. This operation supports conditional HTTP requests. For more +// information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. xMsProperties is optional. User-defined properties to be stored +// with the filesystem, in the format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each +// value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character +// set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed +// if the header is omitted. To merge new and existing properties, first get all existing properties and the current +// E-Tag, then make a conditional request with the E-Tag and include values for all properties. ifModifiedSince is +// optional. A date and time value. Specify this header to perform the operation only if the resource has been modified +// since the specified date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to +// perform the operation only if the resource has not been modified since the specified date and time. +// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client filesystemClient) SetProperties(ctx context.Context, filesystem string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemSetPropertiesResponse, error) { + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.setPropertiesPreparer(filesystem, xMsProperties, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.setPropertiesResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemSetPropertiesResponse), err +} + +// setPropertiesPreparer prepares the SetProperties request. +func (client filesystemClient) setPropertiesPreparer(filesystem string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PATCH", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// setPropertiesResponder handles the response to the SetProperties request. +func (client filesystemClient) setPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemSetPropertiesResponse{rawResponse: resp.Response()}, err +} diff --git a/azbfs/zz_generated_models.go b/azbfs/zz_generated_models.go index b2f8fcb09..4fb458266 100644 --- a/azbfs/zz_generated_models.go +++ b/azbfs/zz_generated_models.go @@ -4,9 +4,11 @@ package azbfs // Changes may cause incorrect behavior and will be lost if the code is regenerated. import ( + "encoding/base64" "io" "net/http" "reflect" + "strconv" "strings" ) @@ -23,802 +25,975 @@ func joinConst(s interface{}, sep string) string { return strings.Join(ss, sep) } -// CreateFilesystemResponse ... -type CreateFilesystemResponse struct { +func validateError(err error) { + if err != nil { + panic(err) + } +} + +// PathGetPropertiesActionType enumerates the values for path get properties action type. +type PathGetPropertiesActionType string + +const ( + // PathGetPropertiesActionGetAccessControl ... + PathGetPropertiesActionGetAccessControl PathGetPropertiesActionType = "getAccessControl" + // PathGetPropertiesActionGetStatus ... + PathGetPropertiesActionGetStatus PathGetPropertiesActionType = "getStatus" + // PathGetPropertiesActionNone represents an empty PathGetPropertiesActionType. + PathGetPropertiesActionNone PathGetPropertiesActionType = "" +) + +// PossiblePathGetPropertiesActionTypeValues returns an array of possible values for the PathGetPropertiesActionType const type. +func PossiblePathGetPropertiesActionTypeValues() []PathGetPropertiesActionType { + return []PathGetPropertiesActionType{PathGetPropertiesActionGetAccessControl, PathGetPropertiesActionGetStatus, PathGetPropertiesActionNone} +} + +// PathLeaseActionType enumerates the values for path lease action type. +type PathLeaseActionType string + +const ( + // PathLeaseActionAcquire ... + PathLeaseActionAcquire PathLeaseActionType = "acquire" + // PathLeaseActionBreak ... + PathLeaseActionBreak PathLeaseActionType = "break" + // PathLeaseActionChange ... + PathLeaseActionChange PathLeaseActionType = "change" + // PathLeaseActionNone represents an empty PathLeaseActionType. + PathLeaseActionNone PathLeaseActionType = "" + // PathLeaseActionRelease ... + PathLeaseActionRelease PathLeaseActionType = "release" + // PathLeaseActionRenew ... + PathLeaseActionRenew PathLeaseActionType = "renew" +) + +// PossiblePathLeaseActionTypeValues returns an array of possible values for the PathLeaseActionType const type. +func PossiblePathLeaseActionTypeValues() []PathLeaseActionType { + return []PathLeaseActionType{PathLeaseActionAcquire, PathLeaseActionBreak, PathLeaseActionChange, PathLeaseActionNone, PathLeaseActionRelease, PathLeaseActionRenew} +} + +// PathRenameModeType enumerates the values for path rename mode type. +type PathRenameModeType string + +const ( + // PathRenameModeLegacy ... + PathRenameModeLegacy PathRenameModeType = "legacy" + // PathRenameModeNone represents an empty PathRenameModeType. + PathRenameModeNone PathRenameModeType = "" + // PathRenameModePosix ... + PathRenameModePosix PathRenameModeType = "posix" +) + +// PossiblePathRenameModeTypeValues returns an array of possible values for the PathRenameModeType const type. +func PossiblePathRenameModeTypeValues() []PathRenameModeType { + return []PathRenameModeType{PathRenameModeLegacy, PathRenameModeNone, PathRenameModePosix} +} + +// PathResourceType enumerates the values for path resource type. +type PathResourceType string + +const ( + // PathResourceDirectory ... + PathResourceDirectory PathResourceType = "directory" + // PathResourceFile ... + PathResourceFile PathResourceType = "file" + // PathResourceNone represents an empty PathResourceType. + PathResourceNone PathResourceType = "" +) + +// PossiblePathResourceTypeValues returns an array of possible values for the PathResourceType const type. +func PossiblePathResourceTypeValues() []PathResourceType { + return []PathResourceType{PathResourceDirectory, PathResourceFile, PathResourceNone} +} + +// PathUpdateActionType enumerates the values for path update action type. +type PathUpdateActionType string + +const ( + // PathUpdateActionAppend ... + PathUpdateActionAppend PathUpdateActionType = "append" + // PathUpdateActionFlush ... + PathUpdateActionFlush PathUpdateActionType = "flush" + // PathUpdateActionNone represents an empty PathUpdateActionType. + PathUpdateActionNone PathUpdateActionType = "" + // PathUpdateActionSetAccessControl ... + PathUpdateActionSetAccessControl PathUpdateActionType = "setAccessControl" + // PathUpdateActionSetProperties ... + PathUpdateActionSetProperties PathUpdateActionType = "setProperties" +) + +// PossiblePathUpdateActionTypeValues returns an array of possible values for the PathUpdateActionType const type. +func PossiblePathUpdateActionTypeValues() []PathUpdateActionType { + return []PathUpdateActionType{PathUpdateActionAppend, PathUpdateActionFlush, PathUpdateActionNone, PathUpdateActionSetAccessControl, PathUpdateActionSetProperties} +} + +// DataLakeStorageError ... +type DataLakeStorageError struct { + // Error - The service error response object. + Error *DataLakeStorageErrorError `json:"error,omitempty"` +} + +// DataLakeStorageErrorError - The service error response object. +type DataLakeStorageErrorError struct { + // Code - The service error code. + Code *string `json:"code,omitempty"` + // Message - The service error message. + Message *string `json:"message,omitempty"` +} + +// Filesystem ... +type Filesystem struct { + Name *string `json:"name,omitempty"` + LastModified *string `json:"lastModified,omitempty"` + ETag *string `json:"eTag,omitempty"` +} + +// FilesystemCreateResponse ... +type FilesystemCreateResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (cfr CreateFilesystemResponse) Response() *http.Response { - return cfr.rawResponse +func (fcr FilesystemCreateResponse) Response() *http.Response { + return fcr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (cfr CreateFilesystemResponse) StatusCode() int { - return cfr.rawResponse.StatusCode +func (fcr FilesystemCreateResponse) StatusCode() int { + return fcr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (cfr CreateFilesystemResponse) Status() string { - return cfr.rawResponse.Status +func (fcr FilesystemCreateResponse) Status() string { + return fcr.rawResponse.Status } // Date returns the value for header Date. -func (cfr CreateFilesystemResponse) Date() string { - return cfr.rawResponse.Header.Get("Date") +func (fcr FilesystemCreateResponse) Date() string { + return fcr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (cfr CreateFilesystemResponse) ETag() string { - return cfr.rawResponse.Header.Get("ETag") +func (fcr FilesystemCreateResponse) ETag() string { + return fcr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (cfr CreateFilesystemResponse) LastModified() string { - return cfr.rawResponse.Header.Get("Last-Modified") +func (fcr FilesystemCreateResponse) LastModified() string { + return fcr.rawResponse.Header.Get("Last-Modified") } // XMsNamespaceEnabled returns the value for header x-ms-namespace-enabled. -func (cfr CreateFilesystemResponse) XMsNamespaceEnabled() string { - return cfr.rawResponse.Header.Get("x-ms-namespace-enabled") +func (fcr FilesystemCreateResponse) XMsNamespaceEnabled() string { + return fcr.rawResponse.Header.Get("x-ms-namespace-enabled") } // XMsRequestID returns the value for header x-ms-request-id. -func (cfr CreateFilesystemResponse) XMsRequestID() string { - return cfr.rawResponse.Header.Get("x-ms-request-id") +func (fcr FilesystemCreateResponse) XMsRequestID() string { + return fcr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (cfr CreateFilesystemResponse) XMsVersion() string { - return cfr.rawResponse.Header.Get("x-ms-version") +func (fcr FilesystemCreateResponse) XMsVersion() string { + return fcr.rawResponse.Header.Get("x-ms-version") } -// CreatePathResponse ... -type CreatePathResponse struct { +// FilesystemDeleteResponse ... +type FilesystemDeleteResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (cpr CreatePathResponse) Response() *http.Response { - return cpr.rawResponse +func (fdr FilesystemDeleteResponse) Response() *http.Response { + return fdr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (cpr CreatePathResponse) StatusCode() int { - return cpr.rawResponse.StatusCode +func (fdr FilesystemDeleteResponse) StatusCode() int { + return fdr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (cpr CreatePathResponse) Status() string { - return cpr.rawResponse.Status +func (fdr FilesystemDeleteResponse) Status() string { + return fdr.rawResponse.Status } -// ContentLength returns the value for header Content-Length. -func (cpr CreatePathResponse) ContentLength() string { - return cpr.rawResponse.Header.Get("Content-Length") +// Date returns the value for header Date. +func (fdr FilesystemDeleteResponse) Date() string { + return fdr.rawResponse.Header.Get("Date") +} + +// XMsRequestID returns the value for header x-ms-request-id. +func (fdr FilesystemDeleteResponse) XMsRequestID() string { + return fdr.rawResponse.Header.Get("x-ms-request-id") +} + +// XMsVersion returns the value for header x-ms-version. +func (fdr FilesystemDeleteResponse) XMsVersion() string { + return fdr.rawResponse.Header.Get("x-ms-version") +} + +// FilesystemGetPropertiesResponse ... +type FilesystemGetPropertiesResponse struct { + rawResponse *http.Response +} + +// Response returns the raw HTTP response object. +func (fgpr FilesystemGetPropertiesResponse) Response() *http.Response { + return fgpr.rawResponse +} + +// StatusCode returns the HTTP status code of the response, e.g. 200. +func (fgpr FilesystemGetPropertiesResponse) StatusCode() int { + return fgpr.rawResponse.StatusCode +} + +// Status returns the HTTP status message of the response, e.g. "200 OK". +func (fgpr FilesystemGetPropertiesResponse) Status() string { + return fgpr.rawResponse.Status } // Date returns the value for header Date. -func (cpr CreatePathResponse) Date() string { - return cpr.rawResponse.Header.Get("Date") +func (fgpr FilesystemGetPropertiesResponse) Date() string { + return fgpr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (cpr CreatePathResponse) ETag() string { - return cpr.rawResponse.Header.Get("ETag") +func (fgpr FilesystemGetPropertiesResponse) ETag() string { + return fgpr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (cpr CreatePathResponse) LastModified() string { - return cpr.rawResponse.Header.Get("Last-Modified") +func (fgpr FilesystemGetPropertiesResponse) LastModified() string { + return fgpr.rawResponse.Header.Get("Last-Modified") } -// XMsContinuation returns the value for header x-ms-continuation. -func (cpr CreatePathResponse) XMsContinuation() string { - return cpr.rawResponse.Header.Get("x-ms-continuation") +// XMsNamespaceEnabled returns the value for header x-ms-namespace-enabled. +func (fgpr FilesystemGetPropertiesResponse) XMsNamespaceEnabled() string { + return fgpr.rawResponse.Header.Get("x-ms-namespace-enabled") +} + +// XMsProperties returns the value for header x-ms-properties. +func (fgpr FilesystemGetPropertiesResponse) XMsProperties() string { + return fgpr.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (cpr CreatePathResponse) XMsRequestID() string { - return cpr.rawResponse.Header.Get("x-ms-request-id") +func (fgpr FilesystemGetPropertiesResponse) XMsRequestID() string { + return fgpr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (cpr CreatePathResponse) XMsVersion() string { - return cpr.rawResponse.Header.Get("x-ms-version") +func (fgpr FilesystemGetPropertiesResponse) XMsVersion() string { + return fgpr.rawResponse.Header.Get("x-ms-version") } -// DeleteFilesystemResponse ... -type DeleteFilesystemResponse struct { +// FilesystemList ... +type FilesystemList struct { rawResponse *http.Response + Filesystems []Filesystem `json:"filesystems,omitempty"` } // Response returns the raw HTTP response object. -func (dfr DeleteFilesystemResponse) Response() *http.Response { - return dfr.rawResponse +func (fl FilesystemList) Response() *http.Response { + return fl.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (dfr DeleteFilesystemResponse) StatusCode() int { - return dfr.rawResponse.StatusCode +func (fl FilesystemList) StatusCode() int { + return fl.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (dfr DeleteFilesystemResponse) Status() string { - return dfr.rawResponse.Status +func (fl FilesystemList) Status() string { + return fl.rawResponse.Status +} + +// ContentType returns the value for header Content-Type. +func (fl FilesystemList) ContentType() string { + return fl.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (dfr DeleteFilesystemResponse) Date() string { - return dfr.rawResponse.Header.Get("Date") +func (fl FilesystemList) Date() string { + return fl.rawResponse.Header.Get("Date") +} + +// XMsContinuation returns the value for header x-ms-continuation. +func (fl FilesystemList) XMsContinuation() string { + return fl.rawResponse.Header.Get("x-ms-continuation") } // XMsRequestID returns the value for header x-ms-request-id. -func (dfr DeleteFilesystemResponse) XMsRequestID() string { - return dfr.rawResponse.Header.Get("x-ms-request-id") +func (fl FilesystemList) XMsRequestID() string { + return fl.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (dfr DeleteFilesystemResponse) XMsVersion() string { - return dfr.rawResponse.Header.Get("x-ms-version") +func (fl FilesystemList) XMsVersion() string { + return fl.rawResponse.Header.Get("x-ms-version") } -// DeletePathResponse ... -type DeletePathResponse struct { +// FilesystemSetPropertiesResponse ... +type FilesystemSetPropertiesResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (dpr DeletePathResponse) Response() *http.Response { - return dpr.rawResponse +func (fspr FilesystemSetPropertiesResponse) Response() *http.Response { + return fspr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (dpr DeletePathResponse) StatusCode() int { - return dpr.rawResponse.StatusCode +func (fspr FilesystemSetPropertiesResponse) StatusCode() int { + return fspr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (dpr DeletePathResponse) Status() string { - return dpr.rawResponse.Status +func (fspr FilesystemSetPropertiesResponse) Status() string { + return fspr.rawResponse.Status } // Date returns the value for header Date. -func (dpr DeletePathResponse) Date() string { - return dpr.rawResponse.Header.Get("Date") +func (fspr FilesystemSetPropertiesResponse) Date() string { + return fspr.rawResponse.Header.Get("Date") } -// XMsContinuation returns the value for header x-ms-continuation. -func (dpr DeletePathResponse) XMsContinuation() string { - return dpr.rawResponse.Header.Get("x-ms-continuation") +// ETag returns the value for header ETag. +func (fspr FilesystemSetPropertiesResponse) ETag() string { + return fspr.rawResponse.Header.Get("ETag") +} + +// LastModified returns the value for header Last-Modified. +func (fspr FilesystemSetPropertiesResponse) LastModified() string { + return fspr.rawResponse.Header.Get("Last-Modified") } // XMsRequestID returns the value for header x-ms-request-id. -func (dpr DeletePathResponse) XMsRequestID() string { - return dpr.rawResponse.Header.Get("x-ms-request-id") +func (fspr FilesystemSetPropertiesResponse) XMsRequestID() string { + return fspr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (dpr DeletePathResponse) XMsVersion() string { - return dpr.rawResponse.Header.Get("x-ms-version") +func (fspr FilesystemSetPropertiesResponse) XMsVersion() string { + return fspr.rawResponse.Header.Get("x-ms-version") } -// ErrorSchema ... -type ErrorSchema struct { - // Error - The service error response object. - Error *ErrorSchemaError `json:"error,omitempty"` -} +// Path ... +type Path struct { + Name *string `json:"name,omitempty"` -// ErrorSchemaError - The service error response object. -type ErrorSchemaError struct { - // Code - The service error code. - Code *string `json:"code,omitempty"` - // Message - The service error message. - Message *string `json:"message,omitempty"` + // begin manual edit to generated code + IsDirectory *bool `json:"isDirectory,string,omitempty"` + // end manual edit + + LastModified *string `json:"lastModified,omitempty"` + ETag *string `json:"eTag,omitempty"` + + // begin manual edit to generated code + ContentLength *int64 `json:"contentLength,string,omitempty"` + // end manual edit + + // begin manual addition to generated code + // TODO: + // (a) How can we verify this will actually work with the JSON that the service will emit, when the service starts to do so? + // (b) One day, consider converting this to use a custom type, that implements TextMarshaller, as has been done + // for the XML-based responses in other SDKs. For now, the decoding from Base64 is up to the caller, and the name is chosen + // to reflect that. + ContentMD5Base64 *string `json:"contentMd5,string,omitempty"` + // end manual addition + + Owner *string `json:"owner,omitempty"` + Group *string `json:"group,omitempty"` + Permissions *string `json:"permissions,omitempty"` } -// GetFilesystemPropertiesResponse ... -type GetFilesystemPropertiesResponse struct { +// PathCreateResponse ... +type PathCreateResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (gfpr GetFilesystemPropertiesResponse) Response() *http.Response { - return gfpr.rawResponse +func (pcr PathCreateResponse) Response() *http.Response { + return pcr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (gfpr GetFilesystemPropertiesResponse) StatusCode() int { - return gfpr.rawResponse.StatusCode +func (pcr PathCreateResponse) StatusCode() int { + return pcr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (gfpr GetFilesystemPropertiesResponse) Status() string { - return gfpr.rawResponse.Status +func (pcr PathCreateResponse) Status() string { + return pcr.rawResponse.Status +} + +// ContentLength returns the value for header Content-Length. +func (pcr PathCreateResponse) ContentLength() int64 { + s := pcr.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i } // Date returns the value for header Date. -func (gfpr GetFilesystemPropertiesResponse) Date() string { - return gfpr.rawResponse.Header.Get("Date") +func (pcr PathCreateResponse) Date() string { + return pcr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (gfpr GetFilesystemPropertiesResponse) ETag() string { - return gfpr.rawResponse.Header.Get("ETag") +func (pcr PathCreateResponse) ETag() string { + return pcr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (gfpr GetFilesystemPropertiesResponse) LastModified() string { - return gfpr.rawResponse.Header.Get("Last-Modified") +func (pcr PathCreateResponse) LastModified() string { + return pcr.rawResponse.Header.Get("Last-Modified") } -// XMsNamespaceEnabled returns the value for header x-ms-namespace-enabled. -func (gfpr GetFilesystemPropertiesResponse) XMsNamespaceEnabled() string { - return gfpr.rawResponse.Header.Get("x-ms-namespace-enabled") +// XMsContinuation returns the value for header x-ms-continuation. +func (pcr PathCreateResponse) XMsContinuation() string { + return pcr.rawResponse.Header.Get("x-ms-continuation") } -// XMsProperties returns the value for header x-ms-properties. -func (gfpr GetFilesystemPropertiesResponse) XMsProperties() string { - return gfpr.rawResponse.Header.Get("x-ms-properties") +// XMsRequestID returns the value for header x-ms-request-id. +func (pcr PathCreateResponse) XMsRequestID() string { + return pcr.rawResponse.Header.Get("x-ms-request-id") +} + +// XMsVersion returns the value for header x-ms-version. +func (pcr PathCreateResponse) XMsVersion() string { + return pcr.rawResponse.Header.Get("x-ms-version") +} + +// PathDeleteResponse ... +type PathDeleteResponse struct { + rawResponse *http.Response +} + +// Response returns the raw HTTP response object. +func (pdr PathDeleteResponse) Response() *http.Response { + return pdr.rawResponse +} + +// StatusCode returns the HTTP status code of the response, e.g. 200. +func (pdr PathDeleteResponse) StatusCode() int { + return pdr.rawResponse.StatusCode +} + +// Status returns the HTTP status message of the response, e.g. "200 OK". +func (pdr PathDeleteResponse) Status() string { + return pdr.rawResponse.Status +} + +// Date returns the value for header Date. +func (pdr PathDeleteResponse) Date() string { + return pdr.rawResponse.Header.Get("Date") +} + +// XMsContinuation returns the value for header x-ms-continuation. +func (pdr PathDeleteResponse) XMsContinuation() string { + return pdr.rawResponse.Header.Get("x-ms-continuation") } // XMsRequestID returns the value for header x-ms-request-id. -func (gfpr GetFilesystemPropertiesResponse) XMsRequestID() string { - return gfpr.rawResponse.Header.Get("x-ms-request-id") +func (pdr PathDeleteResponse) XMsRequestID() string { + return pdr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (gfpr GetFilesystemPropertiesResponse) XMsVersion() string { - return gfpr.rawResponse.Header.Get("x-ms-version") +func (pdr PathDeleteResponse) XMsVersion() string { + return pdr.rawResponse.Header.Get("x-ms-version") } -// GetPathPropertiesResponse ... -type GetPathPropertiesResponse struct { +// PathGetPropertiesResponse ... +type PathGetPropertiesResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (gppr GetPathPropertiesResponse) Response() *http.Response { - return gppr.rawResponse +func (pgpr PathGetPropertiesResponse) Response() *http.Response { + return pgpr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (gppr GetPathPropertiesResponse) StatusCode() int { - return gppr.rawResponse.StatusCode +func (pgpr PathGetPropertiesResponse) StatusCode() int { + return pgpr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (gppr GetPathPropertiesResponse) Status() string { - return gppr.rawResponse.Status +func (pgpr PathGetPropertiesResponse) Status() string { + return pgpr.rawResponse.Status } // AcceptRanges returns the value for header Accept-Ranges. -func (gppr GetPathPropertiesResponse) AcceptRanges() string { - return gppr.rawResponse.Header.Get("Accept-Ranges") +func (pgpr PathGetPropertiesResponse) AcceptRanges() string { + return pgpr.rawResponse.Header.Get("Accept-Ranges") } // CacheControl returns the value for header Cache-Control. -func (gppr GetPathPropertiesResponse) CacheControl() string { - return gppr.rawResponse.Header.Get("Cache-Control") +func (pgpr PathGetPropertiesResponse) CacheControl() string { + return pgpr.rawResponse.Header.Get("Cache-Control") } // ContentDisposition returns the value for header Content-Disposition. -func (gppr GetPathPropertiesResponse) ContentDisposition() string { - return gppr.rawResponse.Header.Get("Content-Disposition") +func (pgpr PathGetPropertiesResponse) ContentDisposition() string { + return pgpr.rawResponse.Header.Get("Content-Disposition") } // ContentEncoding returns the value for header Content-Encoding. -func (gppr GetPathPropertiesResponse) ContentEncoding() string { - return gppr.rawResponse.Header.Get("Content-Encoding") +func (pgpr PathGetPropertiesResponse) ContentEncoding() string { + return pgpr.rawResponse.Header.Get("Content-Encoding") } // ContentLanguage returns the value for header Content-Language. -func (gppr GetPathPropertiesResponse) ContentLanguage() string { - return gppr.rawResponse.Header.Get("Content-Language") +func (pgpr PathGetPropertiesResponse) ContentLanguage() string { + return pgpr.rawResponse.Header.Get("Content-Language") } // ContentLength returns the value for header Content-Length. -func (gppr GetPathPropertiesResponse) ContentLength() string { - return gppr.rawResponse.Header.Get("Content-Length") +func (pgpr PathGetPropertiesResponse) ContentLength() int64 { + s := pgpr.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i } +// ContentMD5 returns the value for header Content-MD5. +// begin manual edit to generated code +func (pgpr PathGetPropertiesResponse) ContentMD5() []byte { + // TODO: why did I have to generate this myself, whereas for blob API corresponding function seems to be + // auto-generated from the Swagger? + s := pgpr.rawResponse.Header.Get("Content-MD5") + if s == "" { + return nil + } + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + b = nil + } + return b +} + +// end manual edit to generated code + // ContentRange returns the value for header Content-Range. -func (gppr GetPathPropertiesResponse) ContentRange() string { - return gppr.rawResponse.Header.Get("Content-Range") +func (pgpr PathGetPropertiesResponse) ContentRange() string { + return pgpr.rawResponse.Header.Get("Content-Range") } // ContentType returns the value for header Content-Type. -func (gppr GetPathPropertiesResponse) ContentType() string { - return gppr.rawResponse.Header.Get("Content-Type") +func (pgpr PathGetPropertiesResponse) ContentType() string { + return pgpr.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (gppr GetPathPropertiesResponse) Date() string { - return gppr.rawResponse.Header.Get("Date") +func (pgpr PathGetPropertiesResponse) Date() string { + return pgpr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (gppr GetPathPropertiesResponse) ETag() string { - return gppr.rawResponse.Header.Get("ETag") +func (pgpr PathGetPropertiesResponse) ETag() string { + return pgpr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (gppr GetPathPropertiesResponse) LastModified() string { - return gppr.rawResponse.Header.Get("Last-Modified") +func (pgpr PathGetPropertiesResponse) LastModified() string { + return pgpr.rawResponse.Header.Get("Last-Modified") } // XMsACL returns the value for header x-ms-acl. -func (gppr GetPathPropertiesResponse) XMsACL() string { - return gppr.rawResponse.Header.Get("x-ms-acl") +func (pgpr PathGetPropertiesResponse) XMsACL() string { + return pgpr.rawResponse.Header.Get("x-ms-acl") } // XMsGroup returns the value for header x-ms-group. -func (gppr GetPathPropertiesResponse) XMsGroup() string { - return gppr.rawResponse.Header.Get("x-ms-group") +func (pgpr PathGetPropertiesResponse) XMsGroup() string { + return pgpr.rawResponse.Header.Get("x-ms-group") } // XMsLeaseDuration returns the value for header x-ms-lease-duration. -func (gppr GetPathPropertiesResponse) XMsLeaseDuration() string { - return gppr.rawResponse.Header.Get("x-ms-lease-duration") +func (pgpr PathGetPropertiesResponse) XMsLeaseDuration() string { + return pgpr.rawResponse.Header.Get("x-ms-lease-duration") } // XMsLeaseState returns the value for header x-ms-lease-state. -func (gppr GetPathPropertiesResponse) XMsLeaseState() string { - return gppr.rawResponse.Header.Get("x-ms-lease-state") +func (pgpr PathGetPropertiesResponse) XMsLeaseState() string { + return pgpr.rawResponse.Header.Get("x-ms-lease-state") } // XMsLeaseStatus returns the value for header x-ms-lease-status. -func (gppr GetPathPropertiesResponse) XMsLeaseStatus() string { - return gppr.rawResponse.Header.Get("x-ms-lease-status") +func (pgpr PathGetPropertiesResponse) XMsLeaseStatus() string { + return pgpr.rawResponse.Header.Get("x-ms-lease-status") } // XMsOwner returns the value for header x-ms-owner. -func (gppr GetPathPropertiesResponse) XMsOwner() string { - return gppr.rawResponse.Header.Get("x-ms-owner") +func (pgpr PathGetPropertiesResponse) XMsOwner() string { + return pgpr.rawResponse.Header.Get("x-ms-owner") } // XMsPermissions returns the value for header x-ms-permissions. -func (gppr GetPathPropertiesResponse) XMsPermissions() string { - return gppr.rawResponse.Header.Get("x-ms-permissions") +func (pgpr PathGetPropertiesResponse) XMsPermissions() string { + return pgpr.rawResponse.Header.Get("x-ms-permissions") } // XMsProperties returns the value for header x-ms-properties. -func (gppr GetPathPropertiesResponse) XMsProperties() string { - return gppr.rawResponse.Header.Get("x-ms-properties") +func (pgpr PathGetPropertiesResponse) XMsProperties() string { + return pgpr.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (gppr GetPathPropertiesResponse) XMsRequestID() string { - return gppr.rawResponse.Header.Get("x-ms-request-id") +func (pgpr PathGetPropertiesResponse) XMsRequestID() string { + return pgpr.rawResponse.Header.Get("x-ms-request-id") } // XMsResourceType returns the value for header x-ms-resource-type. -func (gppr GetPathPropertiesResponse) XMsResourceType() string { - return gppr.rawResponse.Header.Get("x-ms-resource-type") +func (pgpr PathGetPropertiesResponse) XMsResourceType() string { + return pgpr.rawResponse.Header.Get("x-ms-resource-type") } // XMsVersion returns the value for header x-ms-version. -func (gppr GetPathPropertiesResponse) XMsVersion() string { - return gppr.rawResponse.Header.Get("x-ms-version") +func (pgpr PathGetPropertiesResponse) XMsVersion() string { + return pgpr.rawResponse.Header.Get("x-ms-version") } -// LeasePathResponse ... -type LeasePathResponse struct { +// PathLeaseResponse ... +type PathLeaseResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (lpr LeasePathResponse) Response() *http.Response { - return lpr.rawResponse +func (plr PathLeaseResponse) Response() *http.Response { + return plr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (lpr LeasePathResponse) StatusCode() int { - return lpr.rawResponse.StatusCode +func (plr PathLeaseResponse) StatusCode() int { + return plr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (lpr LeasePathResponse) Status() string { - return lpr.rawResponse.Status +func (plr PathLeaseResponse) Status() string { + return plr.rawResponse.Status } // Date returns the value for header Date. -func (lpr LeasePathResponse) Date() string { - return lpr.rawResponse.Header.Get("Date") +func (plr PathLeaseResponse) Date() string { + return plr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (lpr LeasePathResponse) ETag() string { - return lpr.rawResponse.Header.Get("ETag") +func (plr PathLeaseResponse) ETag() string { + return plr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (lpr LeasePathResponse) LastModified() string { - return lpr.rawResponse.Header.Get("Last-Modified") +func (plr PathLeaseResponse) LastModified() string { + return plr.rawResponse.Header.Get("Last-Modified") } // XMsLeaseID returns the value for header x-ms-lease-id. -func (lpr LeasePathResponse) XMsLeaseID() string { - return lpr.rawResponse.Header.Get("x-ms-lease-id") +func (plr PathLeaseResponse) XMsLeaseID() string { + return plr.rawResponse.Header.Get("x-ms-lease-id") } // XMsLeaseTime returns the value for header x-ms-lease-time. -func (lpr LeasePathResponse) XMsLeaseTime() string { - return lpr.rawResponse.Header.Get("x-ms-lease-time") -} - -// XMsRequestID returns the value for header x-ms-request-id. -func (lpr LeasePathResponse) XMsRequestID() string { - return lpr.rawResponse.Header.Get("x-ms-request-id") -} - -// XMsVersion returns the value for header x-ms-version. -func (lpr LeasePathResponse) XMsVersion() string { - return lpr.rawResponse.Header.Get("x-ms-version") -} - -// ListEntrySchema ... -type ListEntrySchema struct { - Name *string `json:"name,omitempty"` - IsDirectory *bool `json:"isDirectory,string,omitempty"` - LastModified *string `json:"lastModified,omitempty"` - ETag *string `json:"eTag,omitempty"` - ContentLength *int64 `json:"contentLength,string,omitempty"` - Owner *string `json:"owner,omitempty"` - Group *string `json:"group,omitempty"` - Permissions *string `json:"permissions,omitempty"` -} - -// ListFilesystemEntry ... -type ListFilesystemEntry struct { - Name *string `json:"name,omitempty"` - LastModified *string `json:"lastModified,omitempty"` - ETag *string `json:"eTag,omitempty"` -} - -// ListFilesystemSchema ... -type ListFilesystemSchema struct { - rawResponse *http.Response - Filesystems []ListFilesystemEntry `json:"filesystems,omitempty"` -} - -// Response returns the raw HTTP response object. -func (lfs ListFilesystemSchema) Response() *http.Response { - return lfs.rawResponse -} - -// StatusCode returns the HTTP status code of the response, e.g. 200. -func (lfs ListFilesystemSchema) StatusCode() int { - return lfs.rawResponse.StatusCode -} - -// Status returns the HTTP status message of the response, e.g. "200 OK". -func (lfs ListFilesystemSchema) Status() string { - return lfs.rawResponse.Status -} - -// ContentType returns the value for header Content-Type. -func (lfs ListFilesystemSchema) ContentType() string { - return lfs.rawResponse.Header.Get("Content-Type") -} - -// Date returns the value for header Date. -func (lfs ListFilesystemSchema) Date() string { - return lfs.rawResponse.Header.Get("Date") -} - -// XMsContinuation returns the value for header x-ms-continuation. -func (lfs ListFilesystemSchema) XMsContinuation() string { - return lfs.rawResponse.Header.Get("x-ms-continuation") +func (plr PathLeaseResponse) XMsLeaseTime() string { + return plr.rawResponse.Header.Get("x-ms-lease-time") } // XMsRequestID returns the value for header x-ms-request-id. -func (lfs ListFilesystemSchema) XMsRequestID() string { - return lfs.rawResponse.Header.Get("x-ms-request-id") +func (plr PathLeaseResponse) XMsRequestID() string { + return plr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (lfs ListFilesystemSchema) XMsVersion() string { - return lfs.rawResponse.Header.Get("x-ms-version") +func (plr PathLeaseResponse) XMsVersion() string { + return plr.rawResponse.Header.Get("x-ms-version") } -// ListSchema ... -type ListSchema struct { +// PathList ... +type PathList struct { rawResponse *http.Response - Paths []ListEntrySchema `json:"paths,omitempty"` + Paths []Path `json:"paths,omitempty"` } // Response returns the raw HTTP response object. -func (ls ListSchema) Response() *http.Response { - return ls.rawResponse +func (pl PathList) Response() *http.Response { + return pl.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (ls ListSchema) StatusCode() int { - return ls.rawResponse.StatusCode +func (pl PathList) StatusCode() int { + return pl.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (ls ListSchema) Status() string { - return ls.rawResponse.Status +func (pl PathList) Status() string { + return pl.rawResponse.Status } // Date returns the value for header Date. -func (ls ListSchema) Date() string { - return ls.rawResponse.Header.Get("Date") +func (pl PathList) Date() string { + return pl.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (ls ListSchema) ETag() string { - return ls.rawResponse.Header.Get("ETag") +func (pl PathList) ETag() string { + return pl.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (ls ListSchema) LastModified() string { - return ls.rawResponse.Header.Get("Last-Modified") +func (pl PathList) LastModified() string { + return pl.rawResponse.Header.Get("Last-Modified") } // XMsContinuation returns the value for header x-ms-continuation. -func (ls ListSchema) XMsContinuation() string { - return ls.rawResponse.Header.Get("x-ms-continuation") +func (pl PathList) XMsContinuation() string { + return pl.rawResponse.Header.Get("x-ms-continuation") } // XMsRequestID returns the value for header x-ms-request-id. -func (ls ListSchema) XMsRequestID() string { - return ls.rawResponse.Header.Get("x-ms-request-id") +func (pl PathList) XMsRequestID() string { + return pl.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (ls ListSchema) XMsVersion() string { - return ls.rawResponse.Header.Get("x-ms-version") +func (pl PathList) XMsVersion() string { + return pl.rawResponse.Header.Get("x-ms-version") } -// ReadPathResponse ... -type ReadPathResponse struct { +// PathUpdateResponse ... +type PathUpdateResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (rpr ReadPathResponse) Response() *http.Response { - return rpr.rawResponse +func (pur PathUpdateResponse) Response() *http.Response { + return pur.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (rpr ReadPathResponse) StatusCode() int { - return rpr.rawResponse.StatusCode +func (pur PathUpdateResponse) StatusCode() int { + return pur.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (rpr ReadPathResponse) Status() string { - return rpr.rawResponse.Status -} - -// Body returns the raw HTTP response object's Body. -func (rpr ReadPathResponse) Body() io.ReadCloser { - return rpr.rawResponse.Body +func (pur PathUpdateResponse) Status() string { + return pur.rawResponse.Status } // AcceptRanges returns the value for header Accept-Ranges. -func (rpr ReadPathResponse) AcceptRanges() string { - return rpr.rawResponse.Header.Get("Accept-Ranges") +func (pur PathUpdateResponse) AcceptRanges() string { + return pur.rawResponse.Header.Get("Accept-Ranges") } // CacheControl returns the value for header Cache-Control. -func (rpr ReadPathResponse) CacheControl() string { - return rpr.rawResponse.Header.Get("Cache-Control") +func (pur PathUpdateResponse) CacheControl() string { + return pur.rawResponse.Header.Get("Cache-Control") } // ContentDisposition returns the value for header Content-Disposition. -func (rpr ReadPathResponse) ContentDisposition() string { - return rpr.rawResponse.Header.Get("Content-Disposition") +func (pur PathUpdateResponse) ContentDisposition() string { + return pur.rawResponse.Header.Get("Content-Disposition") } // ContentEncoding returns the value for header Content-Encoding. -func (rpr ReadPathResponse) ContentEncoding() string { - return rpr.rawResponse.Header.Get("Content-Encoding") +func (pur PathUpdateResponse) ContentEncoding() string { + return pur.rawResponse.Header.Get("Content-Encoding") } // ContentLanguage returns the value for header Content-Language. -func (rpr ReadPathResponse) ContentLanguage() string { - return rpr.rawResponse.Header.Get("Content-Language") +func (pur PathUpdateResponse) ContentLanguage() string { + return pur.rawResponse.Header.Get("Content-Language") } // ContentLength returns the value for header Content-Length. -func (rpr ReadPathResponse) ContentLength() string { - return rpr.rawResponse.Header.Get("Content-Length") +func (pur PathUpdateResponse) ContentLength() int64 { + s := pur.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i } // ContentRange returns the value for header Content-Range. -func (rpr ReadPathResponse) ContentRange() string { - return rpr.rawResponse.Header.Get("Content-Range") +func (pur PathUpdateResponse) ContentRange() string { + return pur.rawResponse.Header.Get("Content-Range") } // ContentType returns the value for header Content-Type. -func (rpr ReadPathResponse) ContentType() string { - return rpr.rawResponse.Header.Get("Content-Type") +func (pur PathUpdateResponse) ContentType() string { + return pur.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (rpr ReadPathResponse) Date() string { - return rpr.rawResponse.Header.Get("Date") +func (pur PathUpdateResponse) Date() string { + return pur.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (rpr ReadPathResponse) ETag() string { - return rpr.rawResponse.Header.Get("ETag") +func (pur PathUpdateResponse) ETag() string { + return pur.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (rpr ReadPathResponse) LastModified() string { - return rpr.rawResponse.Header.Get("Last-Modified") -} - -// XMsLeaseDuration returns the value for header x-ms-lease-duration. -func (rpr ReadPathResponse) XMsLeaseDuration() string { - return rpr.rawResponse.Header.Get("x-ms-lease-duration") -} - -// XMsLeaseState returns the value for header x-ms-lease-state. -func (rpr ReadPathResponse) XMsLeaseState() string { - return rpr.rawResponse.Header.Get("x-ms-lease-state") -} - -// XMsLeaseStatus returns the value for header x-ms-lease-status. -func (rpr ReadPathResponse) XMsLeaseStatus() string { - return rpr.rawResponse.Header.Get("x-ms-lease-status") +func (pur PathUpdateResponse) LastModified() string { + return pur.rawResponse.Header.Get("Last-Modified") } // XMsProperties returns the value for header x-ms-properties. -func (rpr ReadPathResponse) XMsProperties() string { - return rpr.rawResponse.Header.Get("x-ms-properties") +func (pur PathUpdateResponse) XMsProperties() string { + return pur.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (rpr ReadPathResponse) XMsRequestID() string { - return rpr.rawResponse.Header.Get("x-ms-request-id") -} - -// XMsResourceType returns the value for header x-ms-resource-type. -func (rpr ReadPathResponse) XMsResourceType() string { - return rpr.rawResponse.Header.Get("x-ms-resource-type") +func (pur PathUpdateResponse) XMsRequestID() string { + return pur.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (rpr ReadPathResponse) XMsVersion() string { - return rpr.rawResponse.Header.Get("x-ms-version") +func (pur PathUpdateResponse) XMsVersion() string { + return pur.rawResponse.Header.Get("x-ms-version") } -// SetFilesystemPropertiesResponse ... -type SetFilesystemPropertiesResponse struct { +// ReadResponse - Wraps the response from the pathClient.Read method. +type ReadResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (sfpr SetFilesystemPropertiesResponse) Response() *http.Response { - return sfpr.rawResponse +func (rr ReadResponse) Response() *http.Response { + return rr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (sfpr SetFilesystemPropertiesResponse) StatusCode() int { - return sfpr.rawResponse.StatusCode +func (rr ReadResponse) StatusCode() int { + return rr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (sfpr SetFilesystemPropertiesResponse) Status() string { - return sfpr.rawResponse.Status -} - -// Date returns the value for header Date. -func (sfpr SetFilesystemPropertiesResponse) Date() string { - return sfpr.rawResponse.Header.Get("Date") +func (rr ReadResponse) Status() string { + return rr.rawResponse.Status } -// ETag returns the value for header ETag. -func (sfpr SetFilesystemPropertiesResponse) ETag() string { - return sfpr.rawResponse.Header.Get("ETag") -} - -// LastModified returns the value for header Last-Modified. -func (sfpr SetFilesystemPropertiesResponse) LastModified() string { - return sfpr.rawResponse.Header.Get("Last-Modified") -} - -// XMsRequestID returns the value for header x-ms-request-id. -func (sfpr SetFilesystemPropertiesResponse) XMsRequestID() string { - return sfpr.rawResponse.Header.Get("x-ms-request-id") -} - -// XMsVersion returns the value for header x-ms-version. -func (sfpr SetFilesystemPropertiesResponse) XMsVersion() string { - return sfpr.rawResponse.Header.Get("x-ms-version") -} - -// UpdatePathResponse ... -type UpdatePathResponse struct { - rawResponse *http.Response -} - -// Response returns the raw HTTP response object. -func (upr UpdatePathResponse) Response() *http.Response { - return upr.rawResponse -} - -// StatusCode returns the HTTP status code of the response, e.g. 200. -func (upr UpdatePathResponse) StatusCode() int { - return upr.rawResponse.StatusCode -} - -// Status returns the HTTP status message of the response, e.g. "200 OK". -func (upr UpdatePathResponse) Status() string { - return upr.rawResponse.Status +// Body returns the raw HTTP response object's Body. +func (rr ReadResponse) Body() io.ReadCloser { + return rr.rawResponse.Body } // AcceptRanges returns the value for header Accept-Ranges. -func (upr UpdatePathResponse) AcceptRanges() string { - return upr.rawResponse.Header.Get("Accept-Ranges") +func (rr ReadResponse) AcceptRanges() string { + return rr.rawResponse.Header.Get("Accept-Ranges") } // CacheControl returns the value for header Cache-Control. -func (upr UpdatePathResponse) CacheControl() string { - return upr.rawResponse.Header.Get("Cache-Control") +func (rr ReadResponse) CacheControl() string { + return rr.rawResponse.Header.Get("Cache-Control") } // ContentDisposition returns the value for header Content-Disposition. -func (upr UpdatePathResponse) ContentDisposition() string { - return upr.rawResponse.Header.Get("Content-Disposition") +func (rr ReadResponse) ContentDisposition() string { + return rr.rawResponse.Header.Get("Content-Disposition") } // ContentEncoding returns the value for header Content-Encoding. -func (upr UpdatePathResponse) ContentEncoding() string { - return upr.rawResponse.Header.Get("Content-Encoding") +func (rr ReadResponse) ContentEncoding() string { + return rr.rawResponse.Header.Get("Content-Encoding") } // ContentLanguage returns the value for header Content-Language. -func (upr UpdatePathResponse) ContentLanguage() string { - return upr.rawResponse.Header.Get("Content-Language") +func (rr ReadResponse) ContentLanguage() string { + return rr.rawResponse.Header.Get("Content-Language") } // ContentLength returns the value for header Content-Length. -func (upr UpdatePathResponse) ContentLength() string { - return upr.rawResponse.Header.Get("Content-Length") +func (rr ReadResponse) ContentLength() int64 { + s := rr.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i +} + +// ContentMD5 returns the value for header Content-MD5. +func (rr ReadResponse) ContentMD5() string { + return rr.rawResponse.Header.Get("Content-MD5") } // ContentRange returns the value for header Content-Range. -func (upr UpdatePathResponse) ContentRange() string { - return upr.rawResponse.Header.Get("Content-Range") +func (rr ReadResponse) ContentRange() string { + return rr.rawResponse.Header.Get("Content-Range") } // ContentType returns the value for header Content-Type. -func (upr UpdatePathResponse) ContentType() string { - return upr.rawResponse.Header.Get("Content-Type") +func (rr ReadResponse) ContentType() string { + return rr.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (upr UpdatePathResponse) Date() string { - return upr.rawResponse.Header.Get("Date") +func (rr ReadResponse) Date() string { + return rr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (upr UpdatePathResponse) ETag() string { - return upr.rawResponse.Header.Get("ETag") +func (rr ReadResponse) ETag() string { + return rr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (upr UpdatePathResponse) LastModified() string { - return upr.rawResponse.Header.Get("Last-Modified") +func (rr ReadResponse) LastModified() string { + return rr.rawResponse.Header.Get("Last-Modified") +} + +// XMsLeaseDuration returns the value for header x-ms-lease-duration. +func (rr ReadResponse) XMsLeaseDuration() string { + return rr.rawResponse.Header.Get("x-ms-lease-duration") +} + +// XMsLeaseState returns the value for header x-ms-lease-state. +func (rr ReadResponse) XMsLeaseState() string { + return rr.rawResponse.Header.Get("x-ms-lease-state") +} + +// XMsLeaseStatus returns the value for header x-ms-lease-status. +func (rr ReadResponse) XMsLeaseStatus() string { + return rr.rawResponse.Header.Get("x-ms-lease-status") } // XMsProperties returns the value for header x-ms-properties. -func (upr UpdatePathResponse) XMsProperties() string { - return upr.rawResponse.Header.Get("x-ms-properties") +func (rr ReadResponse) XMsProperties() string { + return rr.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (upr UpdatePathResponse) XMsRequestID() string { - return upr.rawResponse.Header.Get("x-ms-request-id") +func (rr ReadResponse) XMsRequestID() string { + return rr.rawResponse.Header.Get("x-ms-request-id") +} + +// XMsResourceType returns the value for header x-ms-resource-type. +func (rr ReadResponse) XMsResourceType() string { + return rr.rawResponse.Header.Get("x-ms-resource-type") } // XMsVersion returns the value for header x-ms-version. -func (upr UpdatePathResponse) XMsVersion() string { - return upr.rawResponse.Header.Get("x-ms-version") +func (rr ReadResponse) XMsVersion() string { + return rr.rawResponse.Header.Get("x-ms-version") } diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go new file mode 100644 index 000000000..83f376910 --- /dev/null +++ b/azbfs/zz_generated_path.go @@ -0,0 +1,868 @@ +package azbfs + +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + // begin manual edit to generated code + "context" + "github.com/Azure/azure-pipeline-go/pipeline" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + // end manual edit +) + +// pathClient is the azure Data Lake Storage provides storage for Hadoop and other big data workloads. +type pathClient struct { + managementClient +} + +// newPathClient creates an instance of the pathClient client. +func newPathClient(url url.URL, p pipeline.Pipeline) pathClient { + return pathClient{newManagementClient(url, p)} +} + +// Create create or rename a file or directory. By default, the destination is overwritten and if the destination +// already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more +// information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// To fail if the destination already exists, use a conditional request with If-None-Match: "*". +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. resource is required only for +// Create File and Create Directory. The value must be "file" or "directory". continuation is optional. When renaming +// a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be +// renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is +// returned in the response, it must be specified in a subsequent invocation of the rename operation to continue +// renaming the directory. mode is optional. Valid only when namespace is enabled. This parameter determines the +// behavior of the rename operation. The value must be "legacy" or "posix", and the default value will be "posix". +// cacheControl is optional. The service stores this value and includes it in the "Cache-Control" response header for +// "Read File" operations for "Read File" operations. contentEncoding is optional. Specifies which content encodings +// have been applied to the file. This value is returned to the client when the "Read File" operation is performed. +// contentLanguage is optional. Specifies the natural language used by the intended audience for the file. +// contentDisposition is optional. The service stores this value and includes it in the "Content-Disposition" response +// header for "Read File" operations. xMsCacheControl is optional. The service stores this value and includes it in +// the "Cache-Control" response header for "Read File" operations. xMsContentType is optional. The service stores this +// value and includes it in the "Content-Type" response header for "Read File" operations. xMsContentEncoding is +// optional. The service stores this value and includes it in the "Content-Encoding" response header for "Read File" +// operations. xMsContentLanguage is optional. The service stores this value and includes it in the "Content-Language" +// response header for "Read File" operations. xMsContentDisposition is optional. The service stores this value and +// includes it in the "Content-Disposition" response header for "Read File" operations. xMsRenameSource is an optional +// file or directory to be renamed. The value must have the following format: "/{filesystem}/{path}". If +// "x-ms-properties" is specified, the properties will overwrite the existing properties; otherwise, the existing +// properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain +// ASCII characters in the ISO-8859-1 character set. xMsLeaseID is optional. A lease ID for the path specified in the +// URI. The path to be overwritten must have an active lease and the lease ID must match. xMsSourceLeaseID is optional +// for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID +// must match. xMsProperties is optional. User-defined properties to be stored with the file or directory, in the +// format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 encoded +// string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. xMsPermissions is +// optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the +// file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The +// sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. +// xMsUmask is optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or +// directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or +// directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. +// For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for +// a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation +// (e.g. 0766). ifMatch is optional. An ETag value. Specify this header to perform the operation only if the +// resource's ETag matches the value specified. The ETag must be specified in quotes. ifNoneMatch is optional. An ETag +// value or the special wildcard ("*") value. Specify this header to perform the operation only if the resource's ETag +// does not match the value specified. The ETag must be specified in quotes. ifModifiedSince is optional. A date and +// time value. Specify this header to perform the operation only if the resource has been modified since the specified +// date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation +// only if the resource has not been modified since the specified date and time. xMsSourceIfMatch is optional. An ETag +// value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. +// The ETag must be specified in quotes. xMsSourceIfNoneMatch is optional. An ETag value or the special wildcard ("*") +// value. Specify this header to perform the rename operation only if the source's ETag does not match the value +// specified. The ETag must be specified in quotes. xMsSourceIfModifiedSince is optional. A date and time value. +// Specify this header to perform the rename operation only if the source has been modified since the specified date +// and time. xMsSourceIfUnmodifiedSince is optional. A date and time value. Specify this header to perform the rename +// operation only if the source has not been modified since the specified date and time. xMsClientRequestID is a UUID +// recorded in the analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value +// in seconds. The period begins when the request is received by the service. If the timeout value elapses before the +// operation completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. +// This is required when using shared key authorization. +func (client pathClient) Create(ctx context.Context, filesystem string, pathParameter string, resource PathResourceType, continuation *string, mode PathRenameModeType, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, xMsUmask *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathCreateResponse, error) { + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: xMsSourceLeaseID, + constraints: []constraint{{target: "xMsSourceLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsSourceLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.createPreparer(filesystem, pathParameter, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, xMsCacheControl, xMsContentType, xMsContentEncoding, xMsContentLanguage, xMsContentDisposition, xMsRenameSource, xMsLeaseID, xMsSourceLeaseID, xMsProperties, xMsPermissions, xMsUmask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsSourceIfMatch, xMsSourceIfNoneMatch, xMsSourceIfModifiedSince, xMsSourceIfUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathCreateResponse), err +} + +// createPreparer prepares the Create request. +func (client pathClient) createPreparer(filesystem string, pathParameter string, resource PathResourceType, continuation *string, mode PathRenameModeType, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, xMsUmask *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PUT", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if resource != PathResourceNone { + params.Set("resource", string(resource)) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if mode != PathRenameModeNone { + params.Set("mode", string(mode)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if cacheControl != nil { + req.Header.Set("Cache-Control", *cacheControl) + } + if contentEncoding != nil { + req.Header.Set("Content-Encoding", *contentEncoding) + } + if contentLanguage != nil { + req.Header.Set("Content-Language", *contentLanguage) + } + if contentDisposition != nil { + req.Header.Set("Content-Disposition", *contentDisposition) + } + if xMsCacheControl != nil { + req.Header.Set("x-ms-cache-control", *xMsCacheControl) + } + if xMsContentType != nil { + req.Header.Set("x-ms-content-type", *xMsContentType) + } + if xMsContentEncoding != nil { + req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) + } + if xMsContentLanguage != nil { + req.Header.Set("x-ms-content-language", *xMsContentLanguage) + } + if xMsContentDisposition != nil { + req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) + } + if xMsRenameSource != nil { + req.Header.Set("x-ms-rename-source", *xMsRenameSource) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsSourceLeaseID != nil { + req.Header.Set("x-ms-source-lease-id", *xMsSourceLeaseID) + } + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsPermissions != nil { + req.Header.Set("x-ms-permissions", *xMsPermissions) + } + if xMsUmask != nil { + req.Header.Set("x-ms-umask", *xMsUmask) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsSourceIfMatch != nil { + req.Header.Set("x-ms-source-if-match", *xMsSourceIfMatch) + } + if xMsSourceIfNoneMatch != nil { + req.Header.Set("x-ms-source-if-none-match", *xMsSourceIfNoneMatch) + } + if xMsSourceIfModifiedSince != nil { + req.Header.Set("x-ms-source-if-modified-since", *xMsSourceIfModifiedSince) + } + if xMsSourceIfUnmodifiedSince != nil { + req.Header.Set("x-ms-source-if-unmodified-since", *xMsSourceIfUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// createResponder handles the response to the Create request. +func (client pathClient) createResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK, http.StatusCreated) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathCreateResponse{rawResponse: resp.Response()}, err +} + +// Delete delete the file or directory. This operation supports conditional HTTP requests. For more information, see +// [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. recursive is required and +// valid only when the resource is a directory. If "true", all paths beneath the directory will be deleted. If "false" +// and the directory is non-empty, an error occurs. continuation is optional. When deleting a directory, the number of +// paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a +// continuation token is returned in this response header. When a continuation token is returned in the response, it +// must be specified in a subsequent invocation of the delete operation to continue deleting the directory. xMsLeaseID +// is the lease ID must be specified if there is an active lease. ifMatch is optional. An ETag value. Specify this +// header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified +// in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to +// perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in +// quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. +// Specify this header to perform the operation only if the resource has not been modified since the specified date and +// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client pathClient) Delete(ctx context.Context, filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathDeleteResponse, error) { + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.deletePreparer(filesystem, pathParameter, recursive, continuation, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathDeleteResponse), err +} + +// deletePreparer prepares the Delete request. +func (client pathClient) deletePreparer(filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("DELETE", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if recursive != nil { + params.Set("recursive", strconv.FormatBool(*recursive)) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// deleteResponder handles the response to the Delete request. +func (client pathClient) deleteResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathDeleteResponse{rawResponse: resp.Response()}, err +} + +// GetProperties get Properties returns all system and user defined properties for a path. Get Status returns all +// system defined properties for a path. Get Access Control List returns the access control list for a path. This +// operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob +// Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. action is optional. If the +// value is "getStatus" only the system defined properties for the path are returned. If the value is +// "getAccessControl" the access control list is returned in the response headers (Hierarchical Namespace must be +// enabled for the account), otherwise the properties are returned. upn is optional. Valid only when Hierarchical +// Namespace is enabled for the account. If "true", the user identity values returned in the x-ms-owner, x-ms-group, +// and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. +// If "false", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that +// group and application Object IDs are not translated because they do not have unique friendly names. xMsLeaseID is +// optional. If this header is specified, the operation will be performed only if both of the following conditions are +// met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path. +// ifMatch is optional. An ETag value. Specify this header to perform the operation only if the resource's ETag +// matches the value specified. The ETag must be specified in quotes. ifNoneMatch is optional. An ETag value or the +// special wildcard ("*") value. Specify this header to perform the operation only if the resource's ETag does not +// match the value specified. The ETag must be specified in quotes. ifModifiedSince is optional. A date and time value. +// Specify this header to perform the operation only if the resource has been modified since the specified date and +// time. ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has not been modified since the specified date and time. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client pathClient) GetProperties(ctx context.Context, filesystem string, pathParameter string, action PathGetPropertiesActionType, upn *bool, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathGetPropertiesResponse, error) { + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.getPropertiesPreparer(filesystem, pathParameter, action, upn, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPropertiesResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathGetPropertiesResponse), err +} + +// getPropertiesPreparer prepares the GetProperties request. +func (client pathClient) getPropertiesPreparer(filesystem string, pathParameter string, action PathGetPropertiesActionType, upn *bool, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("HEAD", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if action != PathGetPropertiesActionNone { + params.Set("action", string(action)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// getPropertiesResponder handles the response to the GetProperties request. +func (client pathClient) getPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathGetPropertiesResponse{rawResponse: resp.Response()}, err +} + +// Lease create and manage a lease to restrict write and delete access to the path. This operation supports conditional +// HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// xMsLeaseAction is there are five lease actions: "acquire", "break", "change", "renew", and "release". Use "acquire" +// and specify the "x-ms-proposed-lease-id" and "x-ms-lease-duration" to acquire a new lease. Use "break" to break an +// existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease +// operation except break and release can be performed on the file. When a lease is successfully broken, the response +// indicates the interval in seconds until a new lease can be acquired. Use "change" and specify the current lease ID +// in "x-ms-lease-id" and the new lease ID in "x-ms-proposed-lease-id" to change the lease ID of an active lease. Use +// "renew" and specify the "x-ms-lease-id" to renew an existing lease. Use "release" and specify the "x-ms-lease-id" to +// release a lease. filesystem is the filesystem identifier. pathParameter is the file or directory path. +// xMsLeaseDuration is the lease duration is required to acquire a lease, and specifies the duration of the lease in +// seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. xMsLeaseBreakPeriod is the +// lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. +// The lease break duration must be between 0 and 60 seconds. xMsLeaseID is required when "x-ms-lease-action" is +// "renew", "change" or "release". For the renew and release actions, this must match the current lease ID. +// xMsProposedLeaseID is required when "x-ms-lease-action" is "acquire" or "change". A lease will be acquired with +// this lease ID if the operation is successful. ifMatch is optional. An ETag value. Specify this header to perform +// the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes. +// ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to perform the +// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. +// ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. +// Specify this header to perform the operation only if the resource has not been modified since the specified date and +// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client pathClient) Lease(ctx context.Context, xMsLeaseAction PathLeaseActionType, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathLeaseResponse, error) { + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: xMsProposedLeaseID, + constraints: []constraint{{target: "xMsProposedLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.leasePreparer(xMsLeaseAction, filesystem, pathParameter, xMsLeaseDuration, xMsLeaseBreakPeriod, xMsLeaseID, xMsProposedLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.leaseResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathLeaseResponse), err +} + +// leasePreparer prepares the Lease request. +func (client pathClient) leasePreparer(xMsLeaseAction PathLeaseActionType, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("POST", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + req.Header.Set("x-ms-lease-action", string(xMsLeaseAction)) + if xMsLeaseDuration != nil { + req.Header.Set("x-ms-lease-duration", strconv.FormatInt(int64(*xMsLeaseDuration), 10)) + } + if xMsLeaseBreakPeriod != nil { + req.Header.Set("x-ms-lease-break-period", strconv.FormatInt(int64(*xMsLeaseBreakPeriod), 10)) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsProposedLeaseID != nil { + req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// leaseResponder handles the response to the Lease request. +func (client pathClient) leaseResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathLeaseResponse{rawResponse: resp.Response()}, err +} + +// Read read the contents of a file. For read operations, range requests are supported. This operation supports +// conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. rangeParameter is the HTTP +// Range request header specifies one or more byte ranges of the resource to be retrieved. xMsLeaseID is optional. If +// this header is specified, the operation will be performed only if both of the following conditions are met: i) the +// path's lease is currently active and ii) the lease ID specified in the request matches that of the path. ifMatch is +// optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value +// specified. The ETag must be specified in quotes. ifNoneMatch is optional. An ETag value or the special wildcard +// ("*") value. Specify this header to perform the operation only if the resource's ETag does not match the value +// specified. The ETag must be specified in quotes. ifModifiedSince is optional. A date and time value. Specify this +// header to perform the operation only if the resource has been modified since the specified date and time. +// ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has not been modified since the specified date and time. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client pathClient) Read(ctx context.Context, filesystem string, pathParameter string, rangeParameter *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ReadResponse, error) { + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.readPreparer(filesystem, pathParameter, rangeParameter, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.readResponder}, req) + if err != nil { + return nil, err + } + return resp.(*ReadResponse), err +} + +// readPreparer prepares the Read request. +func (client pathClient) readPreparer(filesystem string, pathParameter string, rangeParameter *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if rangeParameter != nil { + req.Header.Set("Range", *rangeParameter) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// readResponder handles the response to the Read request. +func (client pathClient) readResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK, http.StatusPartialContent) + if resp == nil { + return nil, err + } + return &ReadResponse{rawResponse: resp.Response()}, err +} + +// Update uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties +// for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This +// operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob +// Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// action is the action must be "append" to upload data to be appended to a file, "flush" to flush previously uploaded +// data to a file, "setProperties" to set the properties of a file or directory, or "setAccessControl" to set the +// owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be +// enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes +// permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are +// mutually exclusive. filesystem is the filesystem identifier. pathParameter is the file or directory path. position +// is this parameter allows the caller to upload data in parallel and control the order in which it is appended to the +// file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to +// the file. The value must be the position where the data is to be appended. Uploaded data is not immediately +// flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter +// must be specified and equal to the length of the file after all data has been written, and there must not be a +// request entity body included with the request. retainUncommittedData is valid only for flush operations. If "true", +// uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after +// the flush operation. The default is false. Data at offsets less than the specified position are written to the +// file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a +// future flush operation. closeParameter is azure Storage Events allow applications to receive notifications when +// files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property +// indicating whether this is the final change to distinguish the difference between an intermediate flush to a file +// stream and the final close of a file stream. The close query parameter is valid only when the action is "flush" and +// change notifications are enabled. If the value of close is "true" and the flush operation completes successfully, +// the service raises a file change notification with a property indicating that this is the final update (the file +// stream has been closed). If "false" a change notification is raised indicating the file has changed. The default is +// false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been +// closed." contentLength is required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the +// length of the request content in bytes for "Append Data". xMsLeaseID is the lease ID must be specified if there is +// an active lease. xMsCacheControl is optional and only valid for flush and set properties operations. The service +// stores this value and includes it in the "Cache-Control" response header for "Read File" operations. xMsContentType +// is optional and only valid for flush and set properties operations. The service stores this value and includes it +// in the "Content-Type" response header for "Read File" operations. xMsContentDisposition is optional and only valid +// for flush and set properties operations. The service stores this value and includes it in the "Content-Disposition" +// response header for "Read File" operations. xMsContentEncoding is optional and only valid for flush and set +// properties operations. The service stores this value and includes it in the "Content-Encoding" response header for +// "Read File" operations. xMsContentLanguage is optional and only valid for flush and set properties operations. The +// service stores this value and includes it in the "Content-Language" response header for "Read File" operations. +// xMsContentMd5 is optional and only valid for "Flush & Set Properties" operations. The service stores this value and +// includes it in the "Content-Md5" response header for "Read & Get Properties" operations. If this property is not +// specified on the request, then the property will be cleared for the file. Subsequent calls to "Read & Get +// Properties" will not return this property unless it is explicitly set on that file again. xMsProperties is optional. +// User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and +// value pairs "n1=v1, n2=v2, ...", where each value is a base64 encoded string. Note that the string may only contain +// ASCII characters in the ISO-8859-1 character set. Valid only for the setProperties operation. If the file or +// directory exists, any properties not included in the list will be removed. All properties are removed if the header +// is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then +// make a conditional request with the E-Tag and include values for all properties. xMsOwner is optional and valid only +// for the setAccessControl operation. Sets the owner of the file or directory. xMsGroup is optional and valid only for +// the setAccessControl operation. Sets the owning group of the file or directory. xMsPermissions is optional and only +// valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the +// file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also +// supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction +// with x-ms-acl. xMsACL is optional and valid only for the setAccessControl operation. Sets POSIX access control +// rights on files and directories. The value is a comma-separated list of access control entries that fully replaces +// the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or +// group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". The scope must be "default" to +// indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the +// access ACL. There are four ACE types: "user" grants rights to the owner or a named user, "group" grants rights to +// the owning group or a named group, "mask" restricts rights granted to named users and the members of groups, and +// "other" grants rights to all users not found in any of the other entries. The user or group identifier is omitted +// for entries of type "mask" and "other". The user or group identifier is also omitted for the owner and owning +// group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the +// second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If +// access is not granted, the '-' character is used to denote that the permission is denied. For example, the following +// ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning +// group, and nothing to everyone else: "user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx". Invalid +// in conjunction with x-ms-permissions. ifMatch is optional for Flush Data and Set Properties, but invalid for Append +// Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value +// specified. The ETag must be specified in quotes. ifNoneMatch is optional for Flush Data and Set Properties, but +// invalid for Append Data. An ETag value or the special wildcard ("*") value. Specify this header to perform the +// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. +// ifModifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. +// Specify this header to perform the operation only if the resource has been modified since the specified date and +// time. ifUnmodifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time +// value. Specify this header to perform the operation only if the resource has not been modified since the specified +// date and time. xHTTPMethodOverride is optional. Override the http verb on the service side. Some older http clients +// do not support PATCH requestBody is valid only for append operations. The data to be uploaded and appended to the +// file. requestBody will be closed upon successful return. Callers should ensure closure when receiving an +// error.xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client pathClient) Update(ctx context.Context, action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathUpdateResponse, error) { + if err := validate([]validation{ + {targetValue: contentLength, + constraints: []constraint{{target: "contentLength", name: null, rule: false, + chain: []constraint{{target: "contentLength", name: inclusiveMinimum, rule: 0, chain: nil}}}}}, + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.updatePreparer(action, filesystem, pathParameter, position, retainUncommittedData, closeParameter, contentLength, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsContentMd5, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xHTTPMethodOverride, body, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.updateResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathUpdateResponse), err +} + +// updatePreparer prepares the Update request. +func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + // begin manual edit to generated code + method := "PATCH" + if xHTTPMethodOverride != nil { + method = "PUT" + } + req, err := pipeline.NewRequest(method, client.url, body) + // end manual edit to generated code + + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("action", string(action)) + if position != nil { + params.Set("position", strconv.FormatInt(*position, 10)) + } + if retainUncommittedData != nil { + params.Set("retainUncommittedData", strconv.FormatBool(*retainUncommittedData)) + } + if closeParameter != nil { + params.Set("close", strconv.FormatBool(*closeParameter)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if contentLength != nil { + req.Header.Set("Content-Length", strconv.FormatInt(*contentLength, 10)) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsCacheControl != nil { + req.Header.Set("x-ms-cache-control", *xMsCacheControl) + } + if xMsContentType != nil { + req.Header.Set("x-ms-content-type", *xMsContentType) + } + if xMsContentDisposition != nil { + req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) + } + if xMsContentEncoding != nil { + req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) + } + if xMsContentLanguage != nil { + req.Header.Set("x-ms-content-language", *xMsContentLanguage) + } + if xMsContentMd5 != nil { + req.Header.Set("x-ms-content-md5", *xMsContentMd5) + } + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsOwner != nil { + req.Header.Set("x-ms-owner", *xMsOwner) + } + if xMsGroup != nil { + req.Header.Set("x-ms-group", *xMsGroup) + } + if xMsPermissions != nil { + req.Header.Set("x-ms-permissions", *xMsPermissions) + } + if xMsACL != nil { + req.Header.Set("x-ms-acl", *xMsACL) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xHTTPMethodOverride != nil { + req.Header.Set("x-http-method-override", *xHTTPMethodOverride) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// updateResponder handles the response to the Update request. +func (client pathClient) updateResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK, http.StatusAccepted) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathUpdateResponse{rawResponse: resp.Response()}, err +} diff --git a/azbfs/zz_generated_responder_policy.go b/azbfs/zz_generated_responder_policy.go index b1d535484..9c35c7723 100644 --- a/azbfs/zz_generated_responder_policy.go +++ b/azbfs/zz_generated_responder_policy.go @@ -55,7 +55,7 @@ func validateResponse(resp pipeline.Response, successStatusCodes ...int) error { defer resp.Response().Body.Close() b, err := ioutil.ReadAll(resp.Response().Body) if err != nil { - return NewResponseError(err, resp.Response(), "failed to read response body") + return err } // the service code, description and details will be populated during unmarshalling responseError := NewResponseError(nil, resp.Response(), resp.Response().Status) diff --git a/azbfs/zz_generated_version.go b/azbfs/zz_generated_version.go index b8072a456..6fe430b24 100644 --- a/azbfs/zz_generated_version.go +++ b/azbfs/zz_generated_version.go @@ -5,7 +5,7 @@ package azbfs // UserAgent returns the UserAgent string to use when sending http.Requests. func UserAgent() string { - return "Azure-SDK-For-Go/0.0.0 azbfs/2018-06-17" + return "Azure-SDK-For-Go/0.0.0 azbfs/2018-11-09" } // Version returns the semantic version (see http://semver.org) of the client. diff --git a/azbfs/zz_response_model.go b/azbfs/zz_response_model.go index 2c2040095..84ab04701 100644 --- a/azbfs/zz_response_model.go +++ b/azbfs/zz_response_model.go @@ -8,262 +8,267 @@ import ( // DirectoryCreateResponse is the CreatePathResponse response type returned for directory specific operations // The type is used to establish difference in the response for file and directory operations since both type of // operations has same response type. -type DirectoryCreateResponse CreatePathResponse +type DirectoryCreateResponse PathCreateResponse // Response returns the raw HTTP response object. func (dcr DirectoryCreateResponse) Response() *http.Response { - return CreatePathResponse(dcr).Response() + return PathCreateResponse(dcr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (dcr DirectoryCreateResponse) StatusCode() int { - return CreatePathResponse(dcr).StatusCode() + return PathCreateResponse(dcr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (dcr DirectoryCreateResponse) Status() string { - return CreatePathResponse(dcr).Status() + return PathCreateResponse(dcr).Status() } // ContentLength returns the value for header Content-Length. -func (dcr DirectoryCreateResponse) ContentLength() string { - return CreatePathResponse(dcr).ContentLength() +func (dcr DirectoryCreateResponse) ContentLength() int64 { + return PathCreateResponse(dcr).ContentLength() } // Date returns the value for header Date. func (dcr DirectoryCreateResponse) Date() string { - return CreatePathResponse(dcr).Date() + return PathCreateResponse(dcr).Date() } // ETag returns the value for header ETag. func (dcr DirectoryCreateResponse) ETag() string { - return CreatePathResponse(dcr).ETag() + return PathCreateResponse(dcr).ETag() } // LastModified returns the value for header Last-Modified. func (dcr DirectoryCreateResponse) LastModified() string { - return CreatePathResponse(dcr).LastModified() + return PathCreateResponse(dcr).LastModified() } // XMsContinuation returns the value for header x-ms-continuation. func (dcr DirectoryCreateResponse) XMsContinuation() string { - return CreatePathResponse(dcr).XMsContinuation() + return PathCreateResponse(dcr).XMsContinuation() } // XMsRequestID returns the value for header x-ms-request-id. func (dcr DirectoryCreateResponse) XMsRequestID() string { - return CreatePathResponse(dcr).XMsRequestID() + return PathCreateResponse(dcr).XMsRequestID() } // XMsVersion returns the value for header x-ms-version. func (dcr DirectoryCreateResponse) XMsVersion() string { - return CreatePathResponse(dcr).XMsVersion() + return PathCreateResponse(dcr).XMsVersion() } // DirectoryDeleteResponse is the DeletePathResponse response type returned for directory specific operations // The type is used to establish difference in the response for file and directory operations since both type of // operations has same response type. -type DirectoryDeleteResponse DeletePathResponse +type DirectoryDeleteResponse PathDeleteResponse // Response returns the raw HTTP response object. func (ddr DirectoryDeleteResponse) Response() *http.Response { - return DeletePathResponse(ddr).Response() + return PathDeleteResponse(ddr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (ddr DirectoryDeleteResponse) StatusCode() int { - return DeletePathResponse(ddr).StatusCode() + return PathDeleteResponse(ddr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (ddr DirectoryDeleteResponse) Status() string { - return DeletePathResponse(ddr).Status() + return PathDeleteResponse(ddr).Status() } // Date returns the value for header Date. func (ddr DirectoryDeleteResponse) Date() string { - return DeletePathResponse(ddr).Date() + return PathDeleteResponse(ddr).Date() } // XMsContinuation returns the value for header x-ms-continuation. func (ddr DirectoryDeleteResponse) XMsContinuation() string { - return DeletePathResponse(ddr).XMsContinuation() + return PathDeleteResponse(ddr).XMsContinuation() } // XMsRequestID returns the value for header x-ms-request-id. func (ddr DirectoryDeleteResponse) XMsRequestID() string { - return DeletePathResponse(ddr).XMsRequestID() + return PathDeleteResponse(ddr).XMsRequestID() } // XMsVersion returns the value for header x-ms-version. func (ddr DirectoryDeleteResponse) XMsVersion() string { - return DeletePathResponse(ddr).XMsVersion() + return PathDeleteResponse(ddr).XMsVersion() } // DirectoryGetPropertiesResponse is the GetPathPropertiesResponse response type returned for directory specific operations // The type is used to establish difference in the response for file and directory operations since both type of // operations has same response type. -type DirectoryGetPropertiesResponse GetPathPropertiesResponse +type DirectoryGetPropertiesResponse PathGetPropertiesResponse // Response returns the raw HTTP response object. func (dgpr DirectoryGetPropertiesResponse) Response() *http.Response { - return GetPathPropertiesResponse(dgpr).Response() + return PathGetPropertiesResponse(dgpr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (dgpr DirectoryGetPropertiesResponse) StatusCode() int { - return GetPathPropertiesResponse(dgpr).StatusCode() + return PathGetPropertiesResponse(dgpr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (dgpr DirectoryGetPropertiesResponse) Status() string { - return GetPathPropertiesResponse(dgpr).Status() + return PathGetPropertiesResponse(dgpr).Status() } // AcceptRanges returns the value for header Accept-Ranges. func (dgpr DirectoryGetPropertiesResponse) AcceptRanges() string { - return GetPathPropertiesResponse(dgpr).AcceptRanges() + return PathGetPropertiesResponse(dgpr).AcceptRanges() } // CacheControl returns the value for header Cache-Control. func (dgpr DirectoryGetPropertiesResponse) CacheControl() string { - return GetPathPropertiesResponse(dgpr).CacheControl() + return PathGetPropertiesResponse(dgpr).CacheControl() } // ContentDisposition returns the value for header Content-Disposition. func (dgpr DirectoryGetPropertiesResponse) ContentDisposition() string { - return GetPathPropertiesResponse(dgpr).ContentDisposition() + return PathGetPropertiesResponse(dgpr).ContentDisposition() } // ContentEncoding returns the value for header Content-Encoding. func (dgpr DirectoryGetPropertiesResponse) ContentEncoding() string { - return GetPathPropertiesResponse(dgpr).ContentEncoding() + return PathGetPropertiesResponse(dgpr).ContentEncoding() } // ContentLanguage returns the value for header Content-Language. func (dgpr DirectoryGetPropertiesResponse) ContentLanguage() string { - return GetPathPropertiesResponse(dgpr).ContentLanguage() + return PathGetPropertiesResponse(dgpr).ContentLanguage() } // ContentLength returns the value for header Content-Length. -func (dgpr DirectoryGetPropertiesResponse) ContentLength() string { - return GetPathPropertiesResponse(dgpr).ContentLength() +func (dgpr DirectoryGetPropertiesResponse) ContentLength() int64 { + return PathGetPropertiesResponse(dgpr).ContentLength() } // ContentRange returns the value for header Content-Range. func (dgpr DirectoryGetPropertiesResponse) ContentRange() string { - return GetPathPropertiesResponse(dgpr).ContentRange() + return PathGetPropertiesResponse(dgpr).ContentRange() } // ContentType returns the value for header Content-Type. func (dgpr DirectoryGetPropertiesResponse) ContentType() string { - return GetPathPropertiesResponse(dgpr).ContentType() + return PathGetPropertiesResponse(dgpr).ContentType() } // Date returns the value for header Date. func (dgpr DirectoryGetPropertiesResponse) Date() string { - return GetPathPropertiesResponse(dgpr).Date() + return PathGetPropertiesResponse(dgpr).Date() } // ETag returns the value for header ETag. func (dgpr DirectoryGetPropertiesResponse) ETag() string { - return GetPathPropertiesResponse(dgpr).ETag() + return PathGetPropertiesResponse(dgpr).ETag() } // LastModified returns the value for header Last-Modified. func (dgpr DirectoryGetPropertiesResponse) LastModified() string { - return GetPathPropertiesResponse(dgpr).LastModified() + return PathGetPropertiesResponse(dgpr).LastModified() } // XMsLeaseDuration returns the value for header x-ms-lease-duration. func (dgpr DirectoryGetPropertiesResponse) XMsLeaseDuration() string { - return GetPathPropertiesResponse(dgpr).XMsLeaseDuration() + return PathGetPropertiesResponse(dgpr).XMsLeaseDuration() } // XMsLeaseState returns the value for header x-ms-lease-state. func (dgpr DirectoryGetPropertiesResponse) XMsLeaseState() string { - return GetPathPropertiesResponse(dgpr).XMsLeaseState() + return PathGetPropertiesResponse(dgpr).XMsLeaseState() } // XMsLeaseStatus returns the value for header x-ms-lease-status. func (dgpr DirectoryGetPropertiesResponse) XMsLeaseStatus() string { - return GetPathPropertiesResponse(dgpr).XMsLeaseStatus() + return PathGetPropertiesResponse(dgpr).XMsLeaseStatus() } // XMsProperties returns the value for header x-ms-properties. func (dgpr DirectoryGetPropertiesResponse) XMsProperties() string { - return GetPathPropertiesResponse(dgpr).XMsProperties() + return PathGetPropertiesResponse(dgpr).XMsProperties() } // XMsRequestID returns the value for header x-ms-request-id. func (dgpr DirectoryGetPropertiesResponse) XMsRequestID() string { - return GetPathPropertiesResponse(dgpr).XMsRequestID() + return PathGetPropertiesResponse(dgpr).XMsRequestID() } // XMsResourceType returns the value for header x-ms-resource-type. func (dgpr DirectoryGetPropertiesResponse) XMsResourceType() string { - return GetPathPropertiesResponse(dgpr).XMsResourceType() + return PathGetPropertiesResponse(dgpr).XMsResourceType() } // XMsVersion returns the value for header x-ms-version. func (dgpr DirectoryGetPropertiesResponse) XMsVersion() string { - return GetPathPropertiesResponse(dgpr).XMsVersion() + return PathGetPropertiesResponse(dgpr).XMsVersion() +} + +// ContentMD5 returns the value for header Content-MD5. +func (dgpr DirectoryGetPropertiesResponse) ContentMD5() []byte { + return PathGetPropertiesResponse(dgpr).ContentMD5() } // DirectoryListResponse is the ListSchema response type. This type declaration is used to implement useful methods on // ListPath response -type DirectoryListResponse ListSchema +type DirectoryListResponse PathList // TODO: Used to by ListPathResponse. Have I changed it to the right thing? // Response returns the raw HTTP response object. func (dlr DirectoryListResponse) Response() *http.Response { - return ListSchema(dlr).Response() + return PathList(dlr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (dlr DirectoryListResponse) StatusCode() int { - return ListSchema(dlr).StatusCode() + return PathList(dlr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (dlr DirectoryListResponse) Status() string { - return ListSchema(dlr).Status() + return PathList(dlr).Status() } // Date returns the value for header Date. func (dlr DirectoryListResponse) Date() string { - return ListSchema(dlr).Date() + return PathList(dlr).Date() } // ETag returns the value for header ETag. func (dlr DirectoryListResponse) ETag() string { - return ListSchema(dlr).ETag() + return PathList(dlr).ETag() } // LastModified returns the value for header Last-Modified. func (dlr DirectoryListResponse) LastModified() string { - return ListSchema(dlr).LastModified() + return PathList(dlr).LastModified() } // XMsContinuation returns the value for header x-ms-continuation. func (dlr DirectoryListResponse) XMsContinuation() string { - return ListSchema(dlr).XMsContinuation() + return PathList(dlr).XMsContinuation() } // XMsRequestID returns the value for header x-ms-request-id. func (dlr DirectoryListResponse) XMsRequestID() string { - return ListSchema(dlr).XMsRequestID() + return PathList(dlr).XMsRequestID() } // XMsVersion returns the value for header x-ms-version. func (dlr DirectoryListResponse) XMsVersion() string { - return ListSchema(dlr).XMsVersion() + return PathList(dlr).XMsVersion() } // Files returns the slice of all Files in ListDirectorySegment Response. // It does not include the sub-directory path -func (dlr *DirectoryListResponse) Files() []ListEntrySchema { - files := []ListEntrySchema{} - lSchema := ListSchema(*dlr) +func (dlr *DirectoryListResponse) Files() []Path { + files := []Path{} + lSchema := PathList(*dlr) for _, path := range lSchema.Paths { if path.IsDirectory != nil && *path.IsDirectory { continue @@ -277,7 +282,7 @@ func (dlr *DirectoryListResponse) Files() []ListEntrySchema { // It does not include the files inside the directory only returns the sub-directories func (dlr *DirectoryListResponse) Directories() []string { var dir []string - lSchema := (ListSchema)(*dlr) + lSchema := (PathList)(*dlr) for _, path := range lSchema.Paths { if path.IsDirectory == nil || (path.IsDirectory != nil && !*path.IsDirectory) { continue @@ -287,9 +292,9 @@ func (dlr *DirectoryListResponse) Directories() []string { return dir } -func (dlr *DirectoryListResponse) FilesAndDirectories() []ListEntrySchema { - var entities []ListEntrySchema - lSchema := (ListSchema)(*dlr) +func (dlr *DirectoryListResponse) FilesAndDirectories() []Path { + var entities []Path + lSchema := (PathList)(*dlr) for _, path := range lSchema.Paths { entities = append(entities, path) } @@ -298,7 +303,7 @@ func (dlr *DirectoryListResponse) FilesAndDirectories() []ListEntrySchema { // DownloadResponse wraps AutoRest generated downloadResponse and helps to provide info for retry. type DownloadResponse struct { - dr *ReadPathResponse + dr *ReadResponse // Fields need for retry. ctx context.Context @@ -347,7 +352,7 @@ func (dr DownloadResponse) ContentLanguage() string { } // ContentLength returns the value for header Content-Length. -func (dr DownloadResponse) ContentLength() string { +func (dr DownloadResponse) ContentLength() int64 { return dr.dr.ContentLength() } diff --git a/cmd/cancel.go b/cmd/cancel.go index 013ddbeaa..985fec741 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -28,6 +28,8 @@ import ( "github.com/spf13/cobra" ) +// TODO should this command be removed? Previously AzCopy was supposed to have an independent backend (out of proc) +// TODO but that's not the plan anymore type rawCancelCmdArgs struct { jobID string } @@ -81,15 +83,15 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("failed to parse user input due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to parse user input due to error " + err.Error()) } err = cooked.process() if err != nil { - glcm.Exit("failed to perform copy command due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to perform copy command due to error " + err.Error()) } - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, // hide features not relevant to BFS // TODO remove after preview release. diff --git a/cmd/copy.go b/cmd/copy.go index 5ecb28b5c..5aa3a37d5 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -84,6 +84,7 @@ type rawCopyCmdArgs struct { contentEncoding string noGuessMimeType bool preserveLastModifiedTime bool + md5ValidationOption string // defines the type of the blob at the destination in case of upload / account to account copy blobType string blockBlobTier string @@ -225,15 +226,19 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { cooked.contentEncoding = raw.contentEncoding cooked.noGuessMimeType = raw.noGuessMimeType cooked.preserveLastModifiedTime = raw.preserveLastModifiedTime + + err = cooked.md5ValidationOption.Parse(raw.md5ValidationOption) + if err != nil { + return cooked, err + } + cooked.background = raw.background cooked.acl = raw.acl cooked.cancelFromStdin = raw.cancelFromStdin // if redirection is triggered, avoid printing any output if cooked.isRedirection() { - cooked.output = common.EOutputFormat.None() - } else { - cooked.output.Parse(raw.output) + glcm.SetOutputFormat(common.EOutputFormat.None()) } // generate a unique job ID @@ -289,6 +294,9 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { return cooked, fmt.Errorf("content-type, content-encoding or metadata is set while copying from sevice to service") } } + if err = validateMd5Option(cooked.md5ValidationOption, cooked.fromTo); err != nil { + return cooked, err + } // If the user has provided some input with excludeBlobType flag, parse the input. if len(raw.excludeBlobType) > 0 { @@ -298,7 +306,7 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { var eBlobType common.BlobType err := eBlobType.Parse(blobType) if err != nil { - return cooked, fmt.Errorf("error parsing the excludeBlobType %s provided with excludeBlobTypeFlag ", blobType) + return cooked, fmt.Errorf("error parsing the exclude-blob-type %s provided with exclude-blob-type flag ", blobType) } cooked.excludeBlobType = append(cooked.excludeBlobType, eBlobType.ToAzBlobType()) } @@ -307,6 +315,15 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { return cooked, nil } +func validateMd5Option(option common.HashValidationOption, fromTo common.FromTo) error { + hasMd5Validation := option != common.DefaultHashValidationOption + isDownload := fromTo.To() == common.ELocation.Local() + if hasMd5Validation && !isDownload { + return fmt.Errorf("md5-validation is set but the job is not a download") + } + return nil +} + // represents the processed copy command input from the user type cookedCopyCmdArgs struct { // from arguments @@ -337,8 +354,8 @@ type cookedCopyCmdArgs struct { contentEncoding string noGuessMimeType bool preserveLastModifiedTime bool + md5ValidationOption common.HashValidationOption background bool - output common.OutputFormat acl string logVerbosity common.LogLevel cancelFromStdin bool @@ -388,7 +405,7 @@ func (cca *cookedCopyCmdArgs) process() error { } // if no error, the operation is now complete - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } return cca.processCopyJobPartOrders() } @@ -509,6 +526,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { Metadata: cca.metadata, NoGuessMimeType: cca.noGuessMimeType, PreserveLastModifiedTime: cca.preserveLastModifiedTime, + MD5ValidationOption: cca.md5ValidationOption, }, // source sas is stripped from the source given by the user and it will not be stored in the part plan file. SourceSAS: cca.sourceSAS, @@ -550,6 +568,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { } } + // TODO remove this copy pasted code during refactoring from := cca.fromTo.From() to := cca.fromTo.To() // Strip the SAS from the source and destination whenever there is SAS exists in URL. @@ -566,6 +585,11 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { blobParts.SAS = azblob.SASQueryParameters{} bUrl := blobParts.URL() cca.source = bUrl.String() + + // set the clean source root + bUrl.Path, _ = gCopyUtil.getRootPathWithoutWildCards(bUrl.Path) + jobPartOrder.SourceRoot = bUrl.String() + case common.ELocation.File(): fromUrl, err := url.Parse(cca.source) if err != nil { @@ -577,6 +601,25 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { fileParts.SAS = azfile.SASQueryParameters{} fUrl := fileParts.URL() cca.source = fUrl.String() + + // set the clean source root + fUrl.Path, _ = gCopyUtil.getRootPathWithoutWildCards(fUrl.Path) + jobPartOrder.SourceRoot = fUrl.String() + + case common.ELocation.Local(): + // If the path separator is '\\', it means + // local path is a windows path + // To avoid path separator check and handling the windows + // path differently, replace the path separator with the + // the linux path separator '/' + if os.PathSeparator == '\\' { + cca.source = strings.Replace(cca.source, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) + } + + jobPartOrder.SourceRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.source) + + default: + jobPartOrder.SourceRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.source) } switch to { @@ -602,29 +645,19 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { fileParts.SAS = azfile.SASQueryParameters{} fUrl := fileParts.URL() cca.destination = fUrl.String() - } - - if from == common.ELocation.Local() { + case common.ELocation.Local(): // If the path separator is '\\', it means // local path is a windows path // To avoid path separator check and handling the windows // path differently, replace the path separator with the // the linux path separator '/' if os.PathSeparator == '\\' { - cca.source = strings.Replace(cca.source, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) + cca.destination = strings.Replace(cca.destination, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) } } - if to == common.ELocation.Local() { - // If the path separator is '\\', it means - // local path is a windows path - // To avoid path separator check and handling the windows - // path differently, replace the path separator with the - // the linux path separator '/' - if os.PathSeparator == '\\' { - cca.destination = strings.Replace(cca.destination, common.OS_PATH_SEPARATOR, "/", -1) - } - } + // set the root destination after it's been cleaned + jobPartOrder.DestinationRoot = cca.destination // lastPartNumber determines the last part number order send for the Job. var lastPartNumber common.PartNumber @@ -698,8 +731,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { // if blocking is specified to false, then another goroutine spawns and wait out the job func (cca *cookedCopyCmdArgs) waitUntilJobCompletion(blocking bool) { // print initial message to indicate that the job is starting - glcm.Info("\nJob " + cca.jobID.String() + " has started\n") - glcm.Info(fmt.Sprintf("Log file is located at: %s/%s.log", azcopyLogPathFolder, cca.jobID)) + glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s/%s.log", azcopyLogPathFolder, cca.jobID))) // initialize the times necessary to track progress cca.jobStartTime = time.Now() @@ -718,10 +750,10 @@ func (cca *cookedCopyCmdArgs) waitUntilJobCompletion(blocking bool) { func (cca *cookedCopyCmdArgs) Cancel(lcm common.LifecycleMgr) { // prompt for confirmation, except when: - // 1. output is in json format + // 1. output is not in text format // 2. azcopy was spawned by another process (cancelFromStdin indicates this) // 3. enumeration is complete - if !(cca.output == common.EOutputFormat.Json() || cca.cancelFromStdin || cca.isEnumerationComplete) { + if !(azcopyOutputFormat != common.EOutputFormat.Text() || cca.cancelFromStdin || cca.isEnumerationComplete) { answer := lcm.Prompt("The source enumeration is not complete, cancelling the job at this point means it cannot be resumed. Please confirm with y/n: ") // read a line from stdin, if the answer is not yes, then abort cancel by returning @@ -732,7 +764,7 @@ func (cca *cookedCopyCmdArgs) Cancel(lcm common.LifecycleMgr) { err := cookedCancelCmdArgs{jobID: cca.jobID}.process() if err != nil { - lcm.Exit("error occurred while cancelling the job "+cca.jobID.String()+". Failed with error "+err.Error(), common.EExitCode.Error()) + lcm.Error("error occurred while cancelling the job " + cca.jobID.String() + ": " + err.Error()) } } @@ -740,78 +772,100 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { // fetch a job status var summary common.ListJobSummaryResponse Rpc(common.ERpcCmd.ListJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() - - // if json output is desired, simply marshal and return - // note that if job is already done, we simply exit - if cca.output == common.EOutputFormat.Json() { - //jsonOutput, err := json.MarshalIndent(summary, "", " ") - jsonOutput, err := json.Marshal(summary) - common.PanicIfErr(err) - - if jobDone { - exitCode := common.EExitCode.Success() - if summary.TransfersFailed > 0 { - exitCode = common.EExitCode.Error() - } - lcm.Exit(string(jsonOutput), exitCode) - } else { - lcm.Info(string(jsonOutput)) - return - } - } + jobDone := summary.JobStatus.IsJobDone() // if json is not desired, and job is done, then we generate a special end message to conclude the job + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job + if jobDone { - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job exitCode := common.EExitCode.Success() if summary.TransfersFailed > 0 { exitCode = common.EExitCode.Error() } - lcm.Exit(fmt.Sprintf( - "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - ste.ToFixed(duration.Minutes(), 4), - summary.TotalTransfers, - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TransfersSkipped, - summary.TotalBytesTransferred, - summary.JobStatus), exitCode) - } - - // if json is not needed, and job is not done, then we generate a message that goes nicely on the same line - // display a scanning keyword if the job is not completely ordered - var scanningString = "" - if !summary.CompleteJobOrdered { - scanningString = "(scanning...)" - } - - // compute the average throughput for the last time interval - bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) - timeElapsed := time.Since(cca.intervalStartTime).Seconds() - throughPut := common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 - - // reset the interval timer and byte count - cca.intervalStartTime = time.Now() - cca.intervalBytesTransferred = summary.BytesOverWire - - // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. - if throughPut == 0 { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, - summary.TotalTransfers, - scanningString)) + + lcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + return fmt.Sprintf( + "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n", + summary.JobID.String(), + ste.ToFixed(duration.Minutes(), 4), + summary.TotalTransfers, + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TransfersSkipped, + summary.TotalBytesTransferred, + summary.JobStatus) + } + }, exitCode) + } + + var computeThroughput = func() float64 { + // compute the average throughput for the last time interval + bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) + timeElapsed := time.Since(cca.intervalStartTime).Seconds() + + // reset the interval timer and byte count + cca.intervalStartTime = time.Now() + cca.intervalBytesTransferred = summary.BytesOverWire + + return common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 + } + + glcm.Progress(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + // if json is not needed, then we generate a message that goes nicely on the same line + // display a scanning keyword if the job is not completely ordered + var scanningString = " (scanning...)" + if summary.CompleteJobOrdered { + scanningString = "" + } + + throughput := computeThroughput() + throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + if throughput == 0 { + // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. + throughputString = "" + } + + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + + return fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s%s%s", + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), + summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, throughputString, diskString) + } + }) +} + +// Is disk speed looking like a constraint on throughput? Ignore the first little-while, +// to give an (arbitrary) amount of time for things to reach steady-state. +func getPerfDisplayText(perfDiagnosticStrings []string, isDiskConstrained bool, durationOfJob time.Duration) (perfString string, diskString string) { + perfString = "" + if shouldDisplayPerfStates() { + perfString = "[States: " + strings.Join(perfDiagnosticStrings, ", ") + "], " + } + + haveBeenRunningLongEnoughToStabilize := durationOfJob.Seconds() > 30 // this duration is an arbitrary guestimate + if isDiskConstrained && haveBeenRunningLongEnoughToStabilize { + diskString = " (disk may be limiting speed)" } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total %s, 2-sec Throughput (Mb/s): %v", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, summary.TotalTransfers, scanningString, ste.ToFixed(throughPut, 4))) + diskString = "" } + return +} + +func shouldDisplayPerfStates() bool { + return glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != "" } func isStdinPipeIn() (bool, error) { @@ -863,17 +917,15 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("failed to parse user input due to error: "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to parse user input due to error: " + err.Error()) } - if cooked.output == common.EOutputFormat.Text() { - glcm.Info("Scanning...") - } + glcm.Info("Scanning...") cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("failed to perform copy command due to error: "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to perform copy command due to error: " + err.Error()) } glcm.SurrenderControl() @@ -891,14 +943,13 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude these files when copying. Support use of *.") cpCmd.PersistentFlags().BoolVar(&raw.forceWrite, "overwrite", true, "overwrite the conflicting files/blobs at the destination if this flag is set to true.") cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when uploading from local file system.") - cpCmd.PersistentFlags().StringVar(&raw.fromTo, "fromTo", "", "optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.") - cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "excludeBlobType", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+ + cpCmd.PersistentFlags().StringVar(&raw.fromTo, "from-to", "", "optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.") + cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "exclude-blob-type", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+ "this flag is not applicable for copying data from non azure-service to service. More than one blob should be separated by ';' ") // options change how the transfers are performed - cpCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json.") cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") cpCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") - cpCmd.PersistentFlags().StringVar(&raw.blobType, "blobType", "None", "defines the type of blob at the destination. This is used in case of upload / account to account copy") + cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "None", "defines the type of blob at the destination. This is used in case of upload / account to account copy") cpCmd.PersistentFlags().StringVar(&raw.blockBlobTier, "block-blob-tier", "None", "upload block blob to Azure Storage using this blob tier.") cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "upload page blob to Azure Storage using this blob tier.") cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "upload to Azure Storage with these key-value pairs as metadata.") @@ -906,6 +957,9 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.contentEncoding, "content-encoding", "", "upload to Azure Storage using this content encoding.") cpCmd.PersistentFlags().BoolVar(&raw.noGuessMimeType, "no-guess-mime-type", false, "prevents AzCopy from detecting the content-type based on the extension/content of the file.") cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "only available when destination is file system.") + cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") + // TODO: should the previous line list the allowable values? + cpCmd.PersistentFlags().BoolVar(&raw.cancelFromStdin, "cancel-from-stdin", false, "true if user wants to cancel the process by passing 'cancel' "+ "to the standard input. This is mostly used when the application is spawned by another process.") cpCmd.PersistentFlags().BoolVar(&raw.background, "background-op", false, "true if user has to perform the operations as a background operation.") @@ -918,8 +972,5 @@ func init() { // Hide the list-of-files flag since it is implemented only for Storage Explorer. cpCmd.PersistentFlags().MarkHidden("list-of-files") cpCmd.PersistentFlags().MarkHidden("include") - cpCmd.PersistentFlags().MarkHidden("output") - cpCmd.PersistentFlags().MarkHidden("stdIn-enable") - cpCmd.PersistentFlags().MarkHidden("background-op") cpCmd.PersistentFlags().MarkHidden("cancel-from-stdin") } diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index c253f7a4d..eb3f2857b 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -72,6 +72,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: blobLocalPath, LastModifiedTime: blobProperties.LastModified(), SourceSize: blobProperties.ContentLength(), + ContentMD5: blobProperties.ContentMD5(), }, cca) // only one transfer for this Job, dispatch the JobPart err := e.dispatchFinalPart(cca) @@ -150,7 +151,9 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobPath)).String(), Destination: util.generateLocalPath(cca.destination, blobRelativePath), LastModifiedTime: blobProperties.LastModified(), - SourceSize: blobProperties.ContentLength()}, cca) + SourceSize: blobProperties.ContentLength(), + ContentMD5: blobProperties.ContentMD5(), + }, cca) continue } if !cca.recursive { @@ -201,14 +204,16 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name)).String(), Destination: util.generateLocalPath(cca.destination, blobRelativePath), LastModifiedTime: blobInfo.Properties.LastModified, - SourceSize: *blobInfo.Properties.ContentLength}, cca) + SourceSize: *blobInfo.Properties.ContentLength, + ContentMD5: blobInfo.Properties.ContentMD5, + }, cca) } marker = listBlob.NextMarker } } // If there are no transfer to queue up, exit with message if len(e.Transfers) == 0 { - glcm.Exit(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination), 1) + glcm.Error(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination)) return nil } // dispatch the JobPart as Final Part of the Job @@ -290,7 +295,9 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name)).String(), Destination: util.generateLocalPath(cca.destination, blobRelativePath), LastModifiedTime: blobInfo.Properties.LastModified, - SourceSize: *blobInfo.Properties.ContentLength}, cca) + SourceSize: *blobInfo.Properties.ContentLength, + ContentMD5: blobInfo.Properties.ContentMD5, + }, cca) } marker = listBlob.NextMarker } diff --git a/cmd/copyDownloadBlobFSEnumerator.go b/cmd/copyDownloadBlobFSEnumerator.go index 7879a00d8..cc9c20930 100644 --- a/cmd/copyDownloadBlobFSEnumerator.go +++ b/cmd/copyDownloadBlobFSEnumerator.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "encoding/base64" "errors" "fmt" "net/url" @@ -11,8 +12,6 @@ import ( "strings" - "strconv" - "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-azcopy/common" ) @@ -55,10 +54,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { destination = cca.destination } - fileSize, err := strconv.ParseInt(props.ContentLength(), 10, 64) - if err != nil { - panic(err) - } + fileSize := props.ContentLength() // Queue the transfer e.addTransfer(common.CopyTransfer{ @@ -66,6 +62,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: destination, LastModifiedTime: e.parseLmt(props.LastModified()), SourceSize: fileSize, + ContentMD5: props.ContentMD5(), }, cca) return e.dispatchFinalPart(cca) @@ -100,10 +97,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { fileURL := azbfs.NewFileURL(tempURLPartsExtension.URL(), p) if fileProperties, err := fileURL.GetProperties(ctx); err == nil && strings.EqualFold(fileProperties.XMsResourceType(), "file") { // file exists - fileSize, err := strconv.ParseInt(fileProperties.ContentLength(), 10, 64) - if err != nil { - panic(err) - } + fileSize := fileProperties.ContentLength() // assembling the file relative path fileRelativePath := fileOrDir @@ -119,7 +113,9 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: srcURL.String(), Destination: util.generateLocalPath(cca.destination, fileRelativePath), LastModifiedTime: e.parseLmt(fileProperties.LastModified()), - SourceSize: fileSize}, cca) + SourceSize: fileSize, + ContentMD5: fileProperties.ContentMD5(), + }, cca) continue } @@ -133,10 +129,10 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { err := enumerateFilesInADLSGen2Directory( ctx, dirURL, - func(fileItem azbfs.ListEntrySchema) bool { // filter always return true in this case + func(fileItem azbfs.Path) bool { // filter always return true in this case return true }, - func(fileItem azbfs.ListEntrySchema) error { + func(fileItem azbfs.Path) error { relativePath := strings.Replace(*fileItem.Name, parentSourcePath, "", 1) if len(relativePath) > 0 && relativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { relativePath = relativePath[1:] @@ -147,6 +143,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: util.generateLocalPath(cca.destination, relativePath), LastModifiedTime: e.parseLmt(*fileItem.LastModified), SourceSize: *fileItem.ContentLength, + ContentMD5: getContentMd5(ctx, directoryURL, fileItem, cca.md5ValidationOption), }, cca) }, ) @@ -157,7 +154,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { } // If there are no transfer to queue up, exit with message if len(e.Transfers) == 0 { - glcm.Exit(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination), 1) + glcm.Error(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination)) return nil } // dispatch the JobPart as Final Part of the Job @@ -182,7 +179,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { for { dListResp, err := directoryURL.ListDirectorySegment(ctx, &continuationMarker, true) if err != nil { - return fmt.Errorf("error listing the files inside the given source url %s", directoryURL.String()) + return fmt.Errorf("error listing the files inside the given source url %s: %s", directoryURL.String(), err.Error()) } // get only the files inside the given path @@ -194,6 +191,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: util.generateLocalPath(cca.destination, util.getRelativePath(fsUrlParts.DirectoryOrFilePath, *path.Name)), LastModifiedTime: e.parseLmt(*path.LastModified), SourceSize: *path.ContentLength, + ContentMD5: getContentMd5(ctx, directoryURL, path, cca.md5ValidationOption), }, cca) } @@ -214,6 +212,37 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { return nil } +func getContentMd5(ctx context.Context, directoryURL azbfs.DirectoryURL, file azbfs.Path, md5ValidationOption common.HashValidationOption) []byte { + if md5ValidationOption == common.EHashValidationOption.NoCheck() { + return nil // not gonna check it, so don't need it + } + + var returnValueForError []byte = nil // If we get an error, we just act like there was no content MD5. If validation is set to fail on error, this will fail the transfer of this file later on (at the time of the MD5 check) + + // convert format of what we have, if we have something in the PathListResponse from Service + if file.ContentMD5Base64 != nil { + value, err := base64.StdEncoding.DecodeString(*file.ContentMD5Base64) + if err != nil { + return returnValueForError + } + return value + } + + // Fall back to making a new round trip to the server + // This is an interim measure, so that we can still validate MD5s even before they are being returned in the server's + // PathList response + // TODO: remove this in a future release, once we know that Service is always returning the MD5s in the PathListResponse. + // Why? Because otherwise, if there's a file with NO MD5, we'll make a round-trip here, but that's pointless if we KNOW that + // that Service is always returning them in the PathListResponse which we've already checked above. + // As at mid-Feb 2019, we don't KNOW that (in fact it's not returning them in the PathListResponse) so we need this code for now. + fileURL := directoryURL.FileSystemURL().NewDirectoryURL(*file.Name) + props, err := fileURL.GetProperties(ctx) + if err != nil { + return returnValueForError + } + return props.ContentMD5() +} + func (e *copyDownloadBlobFSEnumerator) parseLmt(lastModifiedTime string) time.Time { // if last modified time is available, parse it // otherwise use the current time as last modified time diff --git a/cmd/copyDownloadFileEnumerator.go b/cmd/copyDownloadFileEnumerator.go index b6ee37fc5..9357fbf9e 100644 --- a/cmd/copyDownloadFileEnumerator.go +++ b/cmd/copyDownloadFileEnumerator.go @@ -155,7 +155,8 @@ func (e *copyDownloadFileEnumerator) addDownloadFileTransfer(srcURL url.URL, des Source: gCopyUtil.stripSASFromFileShareUrl(srcURL).String(), Destination: destPath, LastModifiedTime: properties.LastModified(), - SourceSize: properties.ContentLength()}, + SourceSize: properties.ContentLength(), + ContentMD5: properties.ContentMD5()}, cca) } diff --git a/cmd/copyEnumeratorHelper.go b/cmd/copyEnumeratorHelper.go index d1f93c21e..1c20e6203 100644 --- a/cmd/copyEnumeratorHelper.go +++ b/cmd/copyEnumeratorHelper.go @@ -16,12 +16,17 @@ import ( // addTransfer accepts a new transfer, if the threshold is reached, dispatch a job part order. func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer, cca *cookedCopyCmdArgs) error { + // Remove the source and destination roots from the path to save space in the plan files + transfer.Source = strings.TrimPrefix(transfer.Source, e.SourceRoot) + transfer.Destination = strings.TrimPrefix(transfer.Destination, e.DestinationRoot) + // dispatch the transfers once the number reaches NumOfFilesPerDispatchJobPart // we do this so that in the case of large transfer, the transfer engine can get started // while the frontend is still gathering more transfers if len(e.Transfers) == NumOfFilesPerDispatchJobPart { shuffleTransfers(e.Transfers) resp := common.CopyJobPartOrderResponse{} + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(e), &resp) if !resp.JobStarted { @@ -35,6 +40,8 @@ func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer e.PartNum++ } + // only append the transfer after we've checked and dispatched a part + // so that there is at least one transfer for the final part e.Transfers = append(e.Transfers, transfer) return nil @@ -219,8 +226,8 @@ func enumerateDirectoriesAndFilesInShare(ctx context.Context, srcDirURL azfile.D ////////////////////////////////////////////////////////////////////////////////////////// // enumerateFilesInADLSGen2Directory enumerates files in ADLS Gen2 directory. func enumerateFilesInADLSGen2Directory(ctx context.Context, directoryURL azbfs.DirectoryURL, - filter func(fileItem azbfs.ListEntrySchema) bool, - callback func(fileItem azbfs.ListEntrySchema) error) error { + filter func(fileItem azbfs.Path) bool, + callback func(fileItem azbfs.Path) error) error { marker := "" for { listDirResp, err := directoryURL.ListDirectorySegment(ctx, &marker, true) diff --git a/cmd/copyEnumeratorHelper_test.go b/cmd/copyEnumeratorHelper_test.go new file mode 100644 index 000000000..645db6ea8 --- /dev/null +++ b/cmd/copyEnumeratorHelper_test.go @@ -0,0 +1,51 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" +) + +type copyEnumeratorHelperTestSuite struct{} + +var _ = chk.Suite(©EnumeratorHelperTestSuite{}) + +func (s *copyEnumeratorHelperTestSuite) TestAddTransferPathRootsTrimmed(c *chk.C) { + // setup + request := common.CopyJobPartOrderRequest{ + SourceRoot: "a/b/", + DestinationRoot: "y/z/", + } + + transfer := common.CopyTransfer{ + Source: "a/b/c.txt", + Destination: "y/z/c.txt", + } + + // execute + err := addTransfer(&request, transfer, &cookedCopyCmdArgs{}) + + // assert + c.Assert(err, chk.IsNil) + c.Assert(request.Transfers[0].Source, chk.Equals, "c.txt") + c.Assert(request.Transfers[0].Destination, chk.Equals, "c.txt") +} diff --git a/cmd/copyUploadEnumerator.go b/cmd/copyUploadEnumerator.go index 8a438aa6b..a3fd88848 100644 --- a/cmd/copyUploadEnumerator.go +++ b/cmd/copyUploadEnumerator.go @@ -87,7 +87,7 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error { if len(cca.listOfFilesToCopy) > 0 { for _, file := range cca.listOfFilesToCopy { tempDestinationURl := *destinationURL - parentSourcePath, _ := util.sourceRootPathWithoutWildCards(cca.source) + parentSourcePath, _ := util.getRootPathWithoutWildCards(cca.source) if len(parentSourcePath) > 0 && parentSourcePath[len(parentSourcePath)-1] == common.AZCOPY_PATH_SEPARATOR_CHAR { parentSourcePath = parentSourcePath[:len(parentSourcePath)-1] } @@ -283,7 +283,7 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error { return err } } - }else{ + } else { glcm.Info(fmt.Sprintf("error %s accessing the filepath %s", err.Error(), fileOrDirectoryPath)) } } @@ -303,7 +303,7 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error { // ffile1, ffile2, then destination for ffile1, ffile2 remotely will be MountedD/MountedF/ffile1 and // MountedD/MountedF/ffile2 func (e *copyUploadEnumerator) getSymlinkTransferList(symlinkPath, source, parentSource, cleanContainerPath string, - destinationUrl *url.URL, cca *cookedCopyCmdArgs) error{ + destinationUrl *url.URL, cca *cookedCopyCmdArgs) error { util := copyHandlerUtil{} // replace the "\\" path separator with "/" separator diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 3d04253ad..d68d9319c 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -119,9 +119,9 @@ func (util copyHandlerUtil) ConstructCommandStringFromArgs() string { } s := strings.Builder{} for _, arg := range args { - // If the argument starts with https, it is either the remote source or remote destination + // If the argument starts with http, it is either the remote source or remote destination // If there exists a signature in the argument string it needs to be redacted - if startsWith(arg, "https") { + if startsWith(arg, "http") { // parse the url argUrl, err := url.Parse(arg) // If there is an error parsing the url, then throw the error @@ -414,10 +414,10 @@ func (util copyHandlerUtil) appendBlobNameToUrl(blobUrlParts azblob.BlobURLParts return blobUrlParts.URL(), blobUrlParts.BlobName } -// sourceRootPathWithoutWildCards returns the directory from path that does not have wildCards +// getRootPathWithoutWildCards returns the directory from path that does not have wildCards // returns the patterns that defines pattern for relativePath of files to the above mentioned directory // For Example: source = C:\User\a*\a1*\*.txt rootDir = C:\User\ pattern = a*\a1*\*.txt -func (util copyHandlerUtil) sourceRootPathWithoutWildCards(path string) (string, string) { +func (util copyHandlerUtil) getRootPathWithoutWildCards(path string) (string, string) { if len(path) == 0 { return path, "*" } diff --git a/cmd/copyUtil_test.go b/cmd/copyUtil_test.go index f5ffbfa79..879feaeac 100644 --- a/cmd/copyUtil_test.go +++ b/cmd/copyUtil_test.go @@ -21,15 +21,10 @@ package cmd import ( - "net/url" - "testing" - chk "gopkg.in/check.v1" + "net/url" ) -// Hookup to the testing framework -func Test(t *testing.T) { chk.TestingT(t) } - type copyUtilTestSuite struct{} var _ = chk.Suite(©UtilTestSuite{}) diff --git a/cmd/credentialUtil.go b/cmd/credentialUtil.go index d87f13022..ca0d29d0b 100644 --- a/cmd/credentialUtil.go +++ b/cmd/credentialUtil.go @@ -246,7 +246,7 @@ func getCredentialType(ctx context.Context, raw rawFromToInfo) (credentialType c default: credentialType = common.ECredentialType.Anonymous() // Log the FromTo types which getCredentialType hasn't solved, in case of miss-use. - glcm.Info(fmt.Sprintf("Use anonymous credential by default for FromTo '%v'", raw.fromTo)) + glcm.Info(fmt.Sprintf("Use anonymous credential by default for from-to '%v'", raw.fromTo)) } return credentialType, nil diff --git a/cmd/env.go b/cmd/env.go index de985d2b0..032a7cda1 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -31,7 +31,7 @@ var envCmd = &cobra.Command{ env.Name, glcm.GetEnvironmentVariable(env), env.Description)) } - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, } diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index ec796f776..9d2cbfc78 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -28,9 +28,9 @@ Copies source data to a destination location. The supported pairs are: Please refer to the examples for more information. Advanced: -Please note that AzCopy automatically detects the Content-Type of files when uploading from local disk, based on file extension or file content(if no extension). +Please note that AzCopy automatically detects the Content Type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). -The built-in lookup table is small but on unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: +The built-in lookup table is small but on Unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: - /etc/mime.types - /etc/apache2/mime.types - /etc/apache/mime.types @@ -38,52 +38,52 @@ The built-in lookup table is small but on unix it is augmented by the local syst On Windows, MIME types are extracted from the registry. This feature can be turned off with the help of a flag. Please refer to the flag section. ` -const copyCmdExample = `Upload a single file with SAS: - - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" +const copyCmdExample = `Upload a single file using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet: +- azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" -Upload a single file with OAuth token, please use login command first if not yet logged in: - - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" +Upload a single file with a SAS: + - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload a single file through piping(block blob only) with SAS: +Upload a single file with a SAS using piping (block blobs only): - cat "/path/to/file.txt" | azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload an entire directory with SAS: +Upload an entire directory with a SAS: - azcopy cp "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Upload only files using wildcards with SAS: +Upload a set of files with a SAS using wildcards: - azcopy cp "/path/*foo/*bar/*.pdf" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" -Upload files and directories using wildcards with SAS: +Upload files and directories with a SAS using wildcards: - azcopy cp "/path/*foo/*bar*" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Download a single file with SAS: - - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "/path/to/file.txt" - -Download a single file with OAuth token, please use login command first if not yet logged in: +Download a single file using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet: - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]" "/path/to/file.txt" -Download a single file through piping(blobs only) with SAS: +Download a single file with a SAS: + - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "/path/to/file.txt" + +Download a single file with a SAS using piping (block blobs only): - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" > "/path/to/file.txt" -Download an entire directory with SAS: +Download an entire directory with a SAS: - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" "/path/to/dir" --recursive=true -Download files using wildcards with SAS: +Download a set of files with a SAS using wildcards: - azcopy cp "https://[account].blob.core.windows.net/[container]/foo*?[SAS]" "/path/to/dir" -Download files and directories using wildcards with SAS: +Download files and directories with a SAS using wildcards: - azcopy cp "https://[account].blob.core.windows.net/[container]/foo*?[SAS]" "/path/to/dir" --recursive=true -Copy a single file with SAS: +Copy a single file between two storage accounts with a SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Copy a single file with OAuth token, please use login command first if not yet logged in and note that OAuth token is used by destination: +Copy a single file between two storage accounts using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet. Note that the same OAuth token is used to access the destination storage account: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]" -Copy an entire directory with SAS: +Copy an entire directory between two storage accounts with a SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Copy an entire account with SAS: +Copy an entire account data to another account with SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net?[SAS]" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true ` @@ -108,7 +108,7 @@ Display information on all jobs.` const showJobsCmdShortDescription = "Show detailed information for the given job ID" const showJobsCmdLongDescription = ` -Show detailed information for the given job ID: if only the job ID is supplied without flag, then the progress summary of the job is returned. +Show detailed information for the given job ID: if only the job ID is supplied without a flag, then the progress summary of the job is returned. If the with-status flag is set, then the list of transfers in the job with the given value will be shown.` const resumeJobsCmdShortDescription = "Resume the existing job with the given job ID" @@ -124,38 +124,38 @@ const listCmdLongDescription = `List the entities in a given resource. Only Blob const listCmdExample = "azcopy list [containerURL]" // ===================================== LOGIN COMMAND ===================================== // -const loginCmdShortDescription = "Log in to Azure Active Directory to access Azure storage resources." +const loginCmdShortDescription = "Log in to Azure Active Directory to access Azure Storage resources." -const loginCmdLongDescription = `Log in to Azure Active Directory to access Azure storage resources. +const loginCmdLongDescription = `Log in to Azure Active Directory to access Azure Storage resources. Note that, to be authorized to your Azure Storage account, you must assign your user 'Storage Blob Data Contributor' role on the Storage account. -This command will cache encrypted login info for current user with OS built-in mechanisms. +This command will cache encrypted login information for current user using the OS built-in mechanisms. Please refer to the examples for more information.` const loginCmdExample = `Log in interactively with default AAD tenant ID set to common: - azcopy login -Log in interactively with specified tenant ID: +Log in interactively with a specified tenant ID: - azcopy login --tenant-id "[TenantID]" Log in using a VM's system-assigned identity: - azcopy login --identity -Log in using a VM's user-assigned identity with Client ID of the service identity: +Log in using a VM's user-assigned identity with a Client ID of the service identity: - azcopy login --identity --identity-client-id "[ServiceIdentityClientID]" -Log in using a VM's user-assigned identity with Object ID of the service identity: +Log in using a VM's user-assigned identity with an Object ID of the service identity: - azcopy login --identity --identity-object-id "[ServiceIdentityObjectID]" -Log in using a VM's user-assigned identity with Resource ID of the service identity: +Log in using a VM's user-assigned identity with a Resource ID of the service identity: - azcopy login --identity --identity-resource-id "/subscriptions//resourcegroups/myRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myID" ` // ===================================== LOGOUT COMMAND ===================================== // -const logoutCmdShortDescription = "Log out to remove access to Azure storage resources." +const logoutCmdShortDescription = "Log out to terminate access to Azure Storage resources." -const logoutCmdLongDescription = `Log out to remove access to Azure storage resources. -This command will remove all the cached login info for current user.` +const logoutCmdLongDescription = `Log out to terminate access to Azure Storage resources. +This command will remove all the cached login information for the current user.` // ===================================== MAKE COMMAND ===================================== // const makeCmdShortDescription = "Create a container/share/filesystem" @@ -167,24 +167,49 @@ const makeCmdExample = ` ` // ===================================== REMOVE COMMAND ===================================== // -const removeCmdShortDescription = "Deletes blobs or files in Azure Storage" +const removeCmdShortDescription = "Delete blobs or files from Azure Storage" -const removeCmdLongDescription = `Deletes blobs or files in Azure Storage.` +const removeCmdLongDescription = `Delete blobs or files from Azure Storage.` // ===================================== SYNC COMMAND ===================================== // -const syncCmdShortDescription = "Replicates source to the destination location" +const syncCmdShortDescription = "Replicate source to the destination location" const syncCmdLongDescription = ` -Replicates source to the destination location. The last modified times are used for comparison. The supported pairs are: - - local <-> Azure Blob (SAS or OAuth authentication) +Replicate a source to a destination location. The last modified times are used for comparison, the file is skipped if the last modified time in the destination is more recent. The supported pairs are: + - local <-> Azure Blob (either SAS or OAuth authentication can be used) + +Please note that the sync command differs from the copy command in several ways: + 0. The recursive flag is on by default. + 1. The source and destination should not contain patterns(such as * or ?). + 2. The include/exclude flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. + 3. If there are files/blobs at the destination that are not present at the source, the user will be prompted to delete them. This prompt can be silenced by using the corresponding flags to automatically answer the deletion question. Advanced: -Please note that AzCopy automatically detects the Content-Type of files when uploading from local disk, based on file extension or file content(if no extension). +Please note that AzCopy automatically detects the Content Type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). -The built-in lookup table is small but on unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: +The built-in lookup table is small but on Unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: - /etc/mime.types - /etc/apache2/mime.types - /etc/apache/mime.types On Windows, MIME types are extracted from the registry. ` + +const syncCmdExample = ` +Sync a single file: + - azcopy sync "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" + +Sync an entire directory including its sub-directories (note that recursive is by default on): + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" + +Sync only the top files inside a directory but not its sub-directories: + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=false + +Sync a subset of files in a directory (ex: only jpg and pdf files, or if the file name is "exactName"): + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --include="*.jpg;*.pdf;exactName" + +Sync an entire directory but exclude certain files from the scope (ex: every file that starts with foo or ends with bar): + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --exclude="foo*;*bar" + +Note: if include/exclude flags are used together, only files matching the include patterns would be looked at, but those matching the exclude patterns would be always be ignored. +` diff --git a/cmd/jobsList.go b/cmd/jobsList.go index b4a8edfab..6f8fe0cc8 100644 --- a/cmd/jobsList.go +++ b/cmd/jobsList.go @@ -21,8 +21,10 @@ package cmd import ( + "encoding/json" "fmt" "sort" + "strings" "time" "github.com/Azure/azure-storage-azcopy/common" @@ -52,9 +54,9 @@ func init() { Run: func(cmd *cobra.Command, args []string) { err := HandleListJobsCommand() if err == nil { - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } else { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } }, } @@ -79,14 +81,24 @@ func PrintExistingJobIds(listJobResponse common.ListJobsResponse) error { // before displaying the jobs, sort them accordingly so that they are displayed in a consistent way sortJobs(listJobResponse.JobIDDetails) - glcm.Info("Existing Jobs ") - for index := 0; index < len(listJobResponse.JobIDDetails); index++ { - jobDetail := listJobResponse.JobIDDetails[index] - glcm.Info(fmt.Sprintf("JobId: %s\nStart Time: %s\nCommand: %s\n", - jobDetail.JobId.String(), - time.Unix(0, jobDetail.StartTime).Format(time.RFC850), - jobDetail.CommandString)) - } + glcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(listJobResponse) + common.PanicIfErr(err) + return string(jsonOutput) + } + + var sb strings.Builder + sb.WriteString("Existing Jobs \n") + for index := 0; index < len(listJobResponse.JobIDDetails); index++ { + jobDetail := listJobResponse.JobIDDetails[index] + sb.WriteString(fmt.Sprintf("JobId: %s\nStart Time: %s\nCommand: %s\n\n", + jobDetail.JobId.String(), + time.Unix(0, jobDetail.StartTime).Format(time.RFC850), + jobDetail.CommandString)) + } + return sb.String() + }, common.EExitCode.Success()) return nil } diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go index f8cfa8574..0be05341b 100644 --- a/cmd/jobsResume.go +++ b/cmd/jobsResume.go @@ -22,6 +22,7 @@ package cmd import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -33,7 +34,8 @@ import ( ) // TODO the behavior of the resume command should be double-checked -// TODO ex: does it output json?? +// TODO figure out how to merge resume job with copy +// TODO the progress reporting code is almost the same as the copy command, the copy-paste should be avoided type resumeJobController struct { // generated jobID common.JobID @@ -54,8 +56,8 @@ type resumeJobController struct { // if blocking is specified to false, then another goroutine spawns and wait out the job func (cca *resumeJobController) waitUntilJobCompletion(blocking bool) { // print initial message to indicate that the job is starting - glcm.Info("\nJob " + cca.jobID.String() + " has started\n") - glcm.Info(fmt.Sprintf("Log file is located at: %s/%s.log", azcopyLogPathFolder, cca.jobID)) + glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s/%s.log", azcopyLogPathFolder, cca.jobID))) + // initialize the times necessary to track progress cca.jobStartTime = time.Now() cca.intervalStartTime = time.Now() @@ -74,7 +76,7 @@ func (cca *resumeJobController) waitUntilJobCompletion(blocking bool) { func (cca *resumeJobController) Cancel(lcm common.LifecycleMgr) { err := cookedCancelCmdArgs{jobID: cca.jobID}.process() if err != nil { - lcm.Exit("error occurred while cancelling the job "+cca.jobID.String()+". Failed with error "+err.Error(), common.EExitCode.Error()) + lcm.Error("error occurred while cancelling the job " + cca.jobID.String() + ". Failed with error " + err.Error()) } } @@ -82,58 +84,79 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) { // fetch a job status var summary common.ListJobSummaryResponse Rpc(common.ERpcCmd.ListJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() + jobDone := summary.JobStatus.IsJobDone() // if json is not desired, and job is done, then we generate a special end message to conclude the job + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job + if jobDone { - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job exitCode := common.EExitCode.Success() if summary.TransfersFailed > 0 { exitCode = common.EExitCode.Error() } - lcm.Exit(fmt.Sprintf( - "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - ste.ToFixed(duration.Minutes(), 4), - summary.TotalTransfers, - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TransfersSkipped, - summary.JobStatus), exitCode) - } - // if json is not needed, and job is not done, then we generate a message that goes nicely on the same line - // display a scanning keyword if the job is not completely ordered - var scanningString = "" - if !summary.CompleteJobOrdered { - scanningString = "(scanning...)" + lcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + return fmt.Sprintf( + "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n", + summary.JobID.String(), + ste.ToFixed(duration.Minutes(), 4), + summary.TotalTransfers, + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TransfersSkipped, + summary.TotalBytesTransferred, + summary.JobStatus) + } + }, exitCode) } - // compute the average throughput for the last time interval - bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) - timeElapsed := time.Since(cca.intervalStartTime).Seconds() - throughPut := common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 + var computeThroughput = func() float64 { + // compute the average throughput for the last time interval + bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) + timeElapsed := time.Since(cca.intervalStartTime).Seconds() - // reset the interval timer and byte count - cca.intervalStartTime = time.Now() - cca.intervalBytesTransferred = summary.BytesOverWire - - // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. - if throughPut == 0 { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, - summary.TotalTransfers, - scanningString)) - } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped %v Total %s, 2-sec Throughput (Mb/s): %v", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, summary.TotalTransfers, scanningString, ste.ToFixed(throughPut, 4))) + // reset the interval timer and byte count + cca.intervalStartTime = time.Now() + cca.intervalBytesTransferred = summary.BytesOverWire + + return common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 } + + glcm.Progress(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + // if json is not needed, then we generate a message that goes nicely on the same line + // display a scanning keyword if the job is not completely ordered + var scanningString = " (scanning...)" + if summary.CompleteJobOrdered { + scanningString = "" + } + + throughput := computeThroughput() + throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + if throughput == 0 { + // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. + throughputString = "" + } + + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + + return fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s%s%s", + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), + summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, throughputString, diskString) + } + }) } func init() { @@ -159,9 +182,9 @@ func init() { Run: func(cmd *cobra.Command, args []string) { err := resumeCmdArgs.process() if err != nil { - glcm.Exit(fmt.Sprintf("failed to perform resume command due to error: %s", err.Error()), common.EExitCode.Error()) + glcm.Error(fmt.Sprintf("failed to perform resume command due to error: %s", err.Error())) } - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, } @@ -232,7 +255,7 @@ func (rca resumeCmdArgs) process() error { &common.GetJobFromToRequest{JobID: jobID}, &getJobFromToResponse) if getJobFromToResponse.ErrorMsg != "" { - glcm.Exit(getJobFromToResponse.ErrorMsg, common.EExitCode.Error()) + glcm.Error(getJobFromToResponse.ErrorMsg) } ctx := context.TODO() @@ -275,7 +298,7 @@ func (rca resumeCmdArgs) process() error { &resumeJobResponse) if !resumeJobResponse.CancelledPauseResumed { - glcm.Exit(resumeJobResponse.ErrorMsg, common.EExitCode.Error()) + glcm.Error(resumeJobResponse.ErrorMsg) } controller := resumeJobController{jobID: jobID} diff --git a/cmd/jobsShow.go b/cmd/jobsShow.go index e7b4e7778..6fbd5ee7e 100644 --- a/cmd/jobsShow.go +++ b/cmd/jobsShow.go @@ -23,6 +23,7 @@ package cmd import ( "errors" "fmt" + "strings" "encoding/json" @@ -33,7 +34,6 @@ import ( type ListReq struct { JobID common.JobID OfStatus string - Output string } func init() { @@ -63,15 +63,12 @@ func init() { listRequest := common.ListRequest{} listRequest.JobID = commandLineInput.JobID listRequest.OfStatus = commandLineInput.OfStatus - err := listRequest.Output.Parse(commandLineInput.Output) - if err != nil { - glcm.Exit(fmt.Errorf("error parsing the given output format %s", commandLineInput.Output).Error(), common.EExitCode.Error()) - } - err = HandleShowCommand(listRequest) + + err := HandleShowCommand(listRequest) if err == nil { - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } else { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } }, } @@ -80,8 +77,6 @@ func init() { // filters shJob.PersistentFlags().StringVar(&commandLineInput.OfStatus, "with-status", "", "only list the transfers of job with this status, available values: Started, Success, Failed") - // filters - shJob.PersistentFlags().StringVar(&commandLineInput.Output, "output", "text", "format of the command's output, the choices include: text, json") } // handles the list command @@ -92,7 +87,7 @@ func HandleShowCommand(listRequest common.ListRequest) error { resp := common.ListJobSummaryResponse{} rpcCmd = common.ERpcCmd.ListJobSummary() Rpc(rpcCmd, &listRequest.JobID, &resp) - PrintJobProgressSummary(listRequest.Output, resp) + PrintJobProgressSummary(resp) } else { lsRequest := common.ListJobTransfersRequest{} lsRequest.JobID = listRequest.JobID @@ -105,67 +100,59 @@ func HandleShowCommand(listRequest common.ListRequest) error { resp := common.ListJobTransfersResponse{} rpcCmd = common.ERpcCmd.ListJobTransfers() Rpc(rpcCmd, lsRequest, &resp) - PrintJobTransfers(listRequest.Output, resp) + PrintJobTransfers(resp) } return nil } // PrintJobTransfers prints the response of listOrder command when list Order command requested the list of specific transfer of an existing job -func PrintJobTransfers(outputForamt common.OutputFormat, listTransfersResponse common.ListJobTransfersResponse) { - if outputForamt == common.EOutputFormat.Json() { - var exitCode = common.EExitCode.Success() - if listTransfersResponse.ErrorMsg != "" { - exitCode = common.EExitCode.Error() - } - //jsonOutput, err := json.MarshalIndent(listTransfersResponse, "", " ") - jsonOutput, err := json.Marshal(listTransfersResponse) - common.PanicIfErr(err) - glcm.Exit(string(jsonOutput), exitCode) - return - } +func PrintJobTransfers(listTransfersResponse common.ListJobTransfersResponse) { if listTransfersResponse.ErrorMsg != "" { - glcm.Exit("request failed with following message "+listTransfersResponse.ErrorMsg, common.EExitCode.Error()) - return + glcm.Error("request failed with following message " + listTransfersResponse.ErrorMsg) } - glcm.Info("----------- Transfers for JobId " + listTransfersResponse.JobID.String() + " -----------") - for index := 0; index < len(listTransfersResponse.Details); index++ { - glcm.Info("transfer--> source: " + listTransfersResponse.Details[index].Src + " destination: " + - listTransfersResponse.Details[index].Dst + " status " + listTransfersResponse.Details[index].TransferStatus.String()) - } + glcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(listTransfersResponse) + common.PanicIfErr(err) + return string(jsonOutput) + } + + var sb strings.Builder + sb.WriteString("----------- Transfers for JobId " + listTransfersResponse.JobID.String() + " -----------\n") + for index := 0; index < len(listTransfersResponse.Details); index++ { + sb.WriteString("transfer--> source: " + listTransfersResponse.Details[index].Src + " destination: " + + listTransfersResponse.Details[index].Dst + " status " + listTransfersResponse.Details[index].TransferStatus.String() + "\n") + } + + return sb.String() + }, common.EExitCode.Success()) } // PrintJobProgressSummary prints the response of listOrder command when listOrder command requested the progress summary of an existing job -func PrintJobProgressSummary(outputFormat common.OutputFormat, summary common.ListJobSummaryResponse) { +func PrintJobProgressSummary(summary common.ListJobSummaryResponse) { + if summary.ErrorMsg != "" { + glcm.Error("list progress summary of job failed because " + summary.ErrorMsg) + } + // Reset the bytes over the wire counter summary.BytesOverWire = 0 - // If the output format is Json, check the summary's error Message. - // If there is an error message, then the exit code is error - // else the exit code is success. - // Marshal the summary and print in the Json format. - if outputFormat == common.EOutputFormat.Json() { - var exitCode = common.EExitCode.Success() - if summary.ErrorMsg != "" { - exitCode = common.EExitCode.Error() + glcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) } - //jsonOutput, err := json.MarshalIndent(summary, "", " ") - jsonOutput, err := json.Marshal(summary) - common.PanicIfErr(err) - glcm.Exit(string(jsonOutput), exitCode) - return - } - if summary.ErrorMsg != "" { - glcm.Exit("list progress summary of job failed because "+summary.ErrorMsg, common.EExitCode.Error()) - } - glcm.Info(fmt.Sprintf( - "\nJob %s summary\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - summary.TotalTransfers, - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TransfersSkipped, - summary.JobStatus, - )) + return fmt.Sprintf( + "\nJob %s summary\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nFinal Job Status: %v\n", + summary.JobID.String(), + summary.TotalTransfers, + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TransfersSkipped, + summary.JobStatus, + ) + }, common.EExitCode.Success()) } diff --git a/cmd/list.go b/cmd/list.go index f6cf97aa9..c9cec3dc4 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -22,7 +22,6 @@ package cmd import ( "context" - "encoding/json" "errors" "fmt" "net/url" @@ -36,7 +35,6 @@ import ( func init() { var sourcePath = "" - var outputRaw = "" // listContainerCmd represents the list container command // listContainer list the blobs inside the container or virtual directory inside the container listContainerCmd := &cobra.Command{ @@ -61,26 +59,23 @@ func init() { // verifying the location type location := inferArgumentLocation(sourcePath) if location != location.Blob() { - glcm.Exit("invalid path passed for listing. given source is of type "+location.String()+" while expect is container / container path ", common.EExitCode.Error()) + glcm.Error("invalid path passed for listing. given source is of type " + location.String() + " while expect is container / container path ") } - var output common.OutputFormat - output.Parse(outputRaw) - err := HandleListContainerCommand(sourcePath, output) + err := HandleListContainerCommand(sourcePath) if err == nil { - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } else { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } }, } rootCmd.AddCommand(listContainerCmd) - listContainerCmd.PersistentFlags().StringVar(&outputRaw, "outputRaw", "text", "format of the command's outputRaw, the choices include: text, json") } // HandleListContainerCommand handles the list container command -func HandleListContainerCommand(source string, outputFormat common.OutputFormat) (err error) { +func HandleListContainerCommand(source string) (err error) { // TODO: Temporarily use context.TODO(), this should be replaced with a root context from main. ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) @@ -152,27 +147,19 @@ func HandleListContainerCommand(source string, outputFormat common.OutputFormat) summary.Blobs = append(summary.Blobs, blobName) } marker = listBlob.NextMarker - printListContainerResponse(&summary, outputFormat) + printListContainerResponse(&summary) } return nil } // printListContainerResponse prints the list container response -func printListContainerResponse(lsResponse *common.ListContainerResponse, outputFormat common.OutputFormat) { +func printListContainerResponse(lsResponse *common.ListContainerResponse) { if len(lsResponse.Blobs) == 0 { return } - if outputFormat == common.EOutputFormat.Json() { - //marshaledData, err := json.MarshalIndent(lsResponse, "", " ") - marshaledData, err := json.Marshal(lsResponse) - if err != nil { - panic(fmt.Errorf("error listing the source. Failed with error %s", err)) - } - glcm.Info(string(marshaledData)) - } else { - for index := 0; index < len(lsResponse.Blobs); index++ { - glcm.Info(lsResponse.Blobs[index]) - } + // TODO determine what's the best way to display the blobs in JSON + // TODO no partner team needs this functionality right now so the blobs are just outputted as info + for index := 0; index < len(lsResponse.Blobs); index++ { + glcm.Info(lsResponse.Blobs[index]) } - lsResponse.Blobs = nil } diff --git a/cmd/make.go b/cmd/make.go index 9cf5e864d..642591596 100644 --- a/cmd/make.go +++ b/cmd/make.go @@ -200,15 +200,17 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cookedArgs, err := rawArgs.cook() if err != nil { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } err = cookedArgs.process() if err != nil { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } - glcm.Exit("Successfully created the resource.", common.EExitCode.Success()) + glcm.Exit(func(format common.OutputFormat) string { + return "Successfully created the resource." + }, common.EExitCode.Success()) }, } diff --git a/cmd/pause.go b/cmd/pause.go index 3ea6fca3a..470c0ef9e 100644 --- a/cmd/pause.go +++ b/cmd/pause.go @@ -27,6 +27,8 @@ import ( "github.com/spf13/cobra" ) +// TODO should this command be removed? Previously AzCopy was supposed to have an independent backend (out of proc) +// TODO but that's not the plan anymore func init() { var commandLineInput = "" @@ -49,7 +51,7 @@ func init() { }, Run: func(cmd *cobra.Command, args []string) { HandlePauseCommand(commandLineInput) - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, // hide features not relevant to BFS // TODO remove after preview release @@ -66,10 +68,12 @@ func HandlePauseCommand(jobIdString string) { jobID, err := common.ParseJobID(jobIdString) if err != nil { // If parsing gives an error, hence it is not a valid JobId format - glcm.Exit("invalid jobId string passed. Failed while parsing string to jobId", common.EExitCode.Error()) + glcm.Error("invalid jobId string passed. Failed while parsing string to jobId") } var pauseJobResponse common.CancelPauseResumeResponse Rpc(common.ERpcCmd.PauseJob(), jobID, &pauseJobResponse) - glcm.Exit("Job "+jobID.String()+" paused successfully", common.EExitCode.Success()) + glcm.Exit(func(format common.OutputFormat) string { + return "Job " + jobID.String() + " paused successfully" + }, common.EExitCode.Success()) } diff --git a/cmd/remove.go b/cmd/remove.go index 672d2e883..cbefbf417 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -51,20 +51,21 @@ func init() { } else { return fmt.Errorf("invalid source type %s pased to delete. azcopy support removing blobs and files only", srcLocationType.String()) } - // Since remove uses the copy command arguments cook, set the blobType to None + // Since remove uses the copy command arguments cook, set the blobType to None and validation option // else parsing the arguments will fail. raw.blobType = common.EBlobType.None().String() + raw.md5ValidationOption = common.DefaultHashValidationOption.String() return nil }, Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("failed to parse user input due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to parse user input due to error " + err.Error()) } cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("failed to perform copy command due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to perform copy command due to error " + err.Error()) } glcm.SurrenderControl() @@ -74,5 +75,4 @@ func init() { deleteCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when deleting from container.") deleteCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") - deleteCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") } diff --git a/cmd/root.go b/cmd/root.go index a15970e9b..65326c601 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,6 +34,8 @@ import ( var azcopyAppPathFolder string var azcopyLogPathFolder string +var outputFormatRaw string +var azcopyOutputFormat common.OutputFormat // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ @@ -41,13 +43,20 @@ var rootCmd = &cobra.Command{ Use: "azcopy", Short: rootCmdShortDescription, Long: rootCmdLongDescription, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + err := azcopyOutputFormat.Parse(outputFormatRaw) + glcm.SetOutputFormat(azcopyOutputFormat) + if err != nil { + return err + } + // spawn a routine to fetch and compare the local application's version against the latest version available // if there's a newer version that can be used, then write the suggestion to stderr // however if this takes too long the message won't get printed // Note: this function is only triggered for non-help commands go detectNewVersion() + return nil }, } @@ -61,16 +70,20 @@ func Execute(azsAppPathFolder, logPathFolder string) { azcopyLogPathFolder = logPathFolder if err := rootCmd.Execute(); err != nil { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } else { // our commands all control their own life explicitly with the lifecycle manager // only help commands reach this point // execute synchronously before exiting detectNewVersion() - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } } +func init() { + rootCmd.PersistentFlags().StringVar(&outputFormatRaw, "output", "text", "format of the command's output, the choices include: text, json.") +} + func detectNewVersion() { const versionMetadataUrl = "https://aka.ms/azcopyv10-version-metadata" @@ -132,7 +145,8 @@ func detectNewVersion() { if v1.OlderThan(*v2) { executablePathSegments := strings.Split(strings.Replace(os.Args[0], "\\", "/", -1), "/") executableName := executablePathSegments[len(executablePathSegments)-1] - // print to stderr instead of stdout, in case the output is in other formats - glcm.Error(executableName + ": A newer version " + remoteVersion + " is available to download\n") + + // output in info mode instead of stderr, as it was crashing CI jobs of some people + glcm.Info(executableName + ": A newer version " + remoteVersion + " is available to download\n") } } diff --git a/cmd/rpc.go b/cmd/rpc.go index 220a30d67..034fed275 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -14,7 +14,6 @@ import ( // Global singleton for sending RPC requests from the frontend to the STE var Rpc = func(cmd common.RpcCmd, request interface{}, response interface{}) { err := inprocSend(cmd, request, response) - //err := NewHttpClient("").send(cmd, request, response) common.PanicIfErr(err) } diff --git a/cmd/sync.go b/cmd/sync.go index 51f845be8..681a11fcc 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -24,6 +24,7 @@ import ( "context" "encoding/json" "fmt" + "path" "time" "net/url" @@ -35,10 +36,10 @@ import ( "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/azure-storage-file-go/azfile" "github.com/spf13/cobra" ) +// a max is set because we cannot buffer infinite amount of destination file info in memory const MaxNumberOfFilesAllowedInSync = 10000000 type rawSyncCmdArgs struct { @@ -46,80 +47,109 @@ type rawSyncCmdArgs struct { dst string recursive bool // options from flags - blockSize uint32 - logVerbosity string - include string - exclude string - followSymlinks bool - output string - // this flag predefines the user-agreement to delete the files in case sync found some files at destination - // which doesn't exists at source. With this flag turned on, user will not be asked for permission before - // deleting the flag. - force bool + blockSize uint32 + logVerbosity string + include string + exclude string + followSymlinks bool + md5ValidationOption string + // this flag indicates the user agreement with respect to deleting the extra files at the destination + // which do not exists at source. With this flag turned on/off, users will not be asked for permission. + // otherwise the user is prompted to make a decision + deleteDestination string +} + +func (raw *rawSyncCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) { + cookedPatterns = make([]string, 0) + rawPatterns := strings.Split(pattern, ";") + for _, pattern := range rawPatterns { + + // skip the empty patterns + if len(pattern) != 0 { + cookedPatterns = append(cookedPatterns, pattern) + } + } + + return +} + +// given a valid URL, parse out the SAS portion +func (raw *rawSyncCmdArgs) separateSasFromURL(rawURL string) (cleanURL string, sas string) { + fromUrl, _ := url.Parse(rawURL) + + // TODO add support for other service URLs + blobParts := azblob.NewBlobURLParts(*fromUrl) + sas = blobParts.SAS.Encode() + + // get clean URL without SAS and trailing / in the path + blobParts.SAS = azblob.SASQueryParameters{} + bUrl := blobParts.URL() + bUrl.Path = strings.TrimSuffix(bUrl.Path, common.AZCOPY_PATH_SEPARATOR_STRING) + cleanURL = bUrl.String() + + return +} + +func (raw *rawSyncCmdArgs) cleanLocalPath(rawPath string) (cleanPath string) { + // if the path separator is '\\', it means + // local path is a windows path + // to avoid path separator check and handling the windows + // path differently, replace the path separator with the + // the linux path separator '/' + if os.PathSeparator == '\\' { + cleanPath = strings.Replace(rawPath, common.OS_PATH_SEPARATOR, "/", -1) + } + cleanPath = path.Clean(rawPath) + return } // validates and transform raw input into cooked input -func (raw rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { +func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { cooked := cookedSyncCmdArgs{} - fromTo := inferFromTo(raw.src, raw.dst) - if fromTo == common.EFromTo.Unknown() { + cooked.fromTo = inferFromTo(raw.src, raw.dst) + if cooked.fromTo == common.EFromTo.Unknown() { return cooked, fmt.Errorf("Unable to infer the source '%s' / destination '%s'. ", raw.src, raw.dst) + } else if cooked.fromTo == common.EFromTo.LocalBlob() { + cooked.source = raw.cleanLocalPath(raw.src) + cooked.destination, cooked.destinationSAS = raw.separateSasFromURL(raw.dst) + } else if cooked.fromTo == common.EFromTo.BlobLocal() { + cooked.source, cooked.sourceSAS = raw.separateSasFromURL(raw.src) + cooked.destination = raw.cleanLocalPath(raw.dst) + } else { + return cooked, fmt.Errorf("source '%s' / destination '%s' combination '%s' not supported for sync command ", raw.src, raw.dst, cooked.fromTo) } - if fromTo != common.EFromTo.LocalBlob() && - fromTo != common.EFromTo.BlobLocal() { - return cooked, fmt.Errorf("source '%s' / destination '%s' combination '%s' not supported for sync command ", raw.src, raw.dst, fromTo) - } - cooked.source = raw.src - cooked.destination = raw.dst - cooked.fromTo = fromTo + // generate a new job ID + cooked.jobID = common.NewJobID() cooked.blockSize = raw.blockSize - cooked.followSymlinks = raw.followSymlinks + cooked.recursive = raw.recursive - err := cooked.logVerbosity.Parse(raw.logVerbosity) + // determine whether we should prompt the user to delete extra files + err := cooked.deleteDestination.Parse(raw.deleteDestination) if err != nil { return cooked, err } - // initialize the include map which contains the list of files to be included - // parse the string passed in include flag - // more than one file are expected to be separated by ';' - cooked.include = make(map[string]int) - if len(raw.include) > 0 { - files := strings.Split(raw.include, ";") - for index := range files { - // If split of the include string leads to an empty string - // not include that string - if len(files[index]) == 0 { - continue - } - cooked.include[files[index]] = index - } + // parse the filter patterns + cooked.include = raw.parsePatterns(raw.include) + cooked.exclude = raw.parsePatterns(raw.exclude) + + err = cooked.logVerbosity.Parse(raw.logVerbosity) + if err != nil { + return cooked, err } - // initialize the exclude map which contains the list of files to be excluded - // parse the string passed in exclude flag - // more than one file are expected to be separated by ';' - cooked.exclude = make(map[string]int) - if len(raw.exclude) > 0 { - files := strings.Split(raw.exclude, ";") - for index := range files { - // If split of the include string leads to an empty string - // not include that string - if len(files[index]) == 0 { - continue - } - cooked.exclude[files[index]] = index - } + err = cooked.md5ValidationOption.Parse(raw.md5ValidationOption) + if err != nil { + return cooked, err + } + if err = validateMd5Option(cooked.md5ValidationOption, cooked.fromTo); err != nil { + return cooked, err } - cooked.recursive = raw.recursive - cooked.output.Parse(raw.output) - cooked.jobID = common.NewJobID() - cooked.force = raw.force return cooked, nil } @@ -129,14 +159,19 @@ type cookedSyncCmdArgs struct { destination string destinationSAS string fromTo common.FromTo + credentialInfo common.CredentialInfo + + // filters recursive bool followSymlinks bool - // options from flags - include map[string]int - exclude map[string]int - blockSize uint32 - logVerbosity common.LogLevel - output common.OutputFormat + include []string + exclude []string + + // options + md5ValidationOption common.HashValidationOption + blockSize uint32 + logVerbosity common.LogLevel + // commandString hold the user given command which is logged to the Job log file commandString string @@ -155,6 +190,7 @@ type cookedSyncCmdArgs struct { // this flag is set by the enumerator // it is useful to indicate whether we are simply waiting for the purpose of cancelling + // this is set to true once the final part has been dispatched isEnumerationComplete bool // defines the scanning status of the sync operation. @@ -167,10 +203,22 @@ type cookedSyncCmdArgs struct { atomicSourceFilesScanned uint64 // defines the number of files listed at the destination and compared. atomicDestinationFilesScanned uint64 - // this flag predefines the user-agreement to delete the files in case sync found some files at destination - // which doesn't exists at source. With this flag turned on, user will not be asked for permission before - // deleting the flag. - force bool + + // deletion count keeps track of how many extra files from the destination were removed + atomicDeletionCount uint32 + + // this flag indicates the user agreement with respect to deleting the extra files at the destination + // which do not exists at source. With this flag turned on/off, users will not be asked for permission. + // otherwise the user is prompted to make a decision + deleteDestination common.DeleteDestination +} + +func (cca *cookedSyncCmdArgs) incrementDeletionCount() { + atomic.AddUint32(&cca.atomicDeletionCount, 1) +} + +func (cca *cookedSyncCmdArgs) getDeletionCount() uint32 { + return atomic.LoadUint32(&cca.atomicDeletionCount) } // setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1 @@ -198,8 +246,7 @@ func (cca *cookedSyncCmdArgs) scanningComplete() bool { // if blocking is specified to false, then another goroutine spawns and wait out the job func (cca *cookedSyncCmdArgs) waitUntilJobCompletion(blocking bool) { // print initial message to indicate that the job is starting - glcm.Info("\nJob " + cca.jobID.String() + " has started\n") - glcm.Info(fmt.Sprintf("Log file is located at: %s/%s.log", azcopyLogPathFolder, cca.jobID)) + glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s/%s.log", azcopyLogPathFolder, cca.jobID))) // initialize the times necessary to track progress cca.jobStartTime = time.Now() @@ -217,10 +264,8 @@ func (cca *cookedSyncCmdArgs) waitUntilJobCompletion(blocking bool) { } func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { - // prompt for confirmation, except when: - // 1. output is in json format - // 2. enumeration is complete - if !(cca.output == common.EOutputFormat.Json() || cca.isEnumerationComplete) { + // prompt for confirmation, except when enumeration is complete + if !cca.isEnumerationComplete { answer := lcm.Prompt("The source enumeration is not complete, cancelling the job at this point means it cannot be resumed. Please confirm with y/n: ") // read a line from stdin, if the answer is not yes, then abort cancel by returning @@ -231,119 +276,155 @@ func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { err := cookedCancelCmdArgs{jobID: cca.jobID}.process() if err != nil { - lcm.Exit("error occurred while cancelling the job "+cca.jobID.String()+". Failed with error "+err.Error(), common.EExitCode.Error()) + lcm.Error("error occurred while cancelling the job " + cca.jobID.String() + ". Failed with error " + err.Error()) } } +type scanningProgressJsonTemplate struct { + FilesScannedAtSource uint64 + FilesScannedAtDestination uint64 +} + +func (cca *cookedSyncCmdArgs) reportScanningProgress(lcm common.LifecycleMgr, throughput float64) { + + lcm.Progress(func(format common.OutputFormat) string { + srcScanned := atomic.LoadUint64(&cca.atomicSourceFilesScanned) + dstScanned := atomic.LoadUint64(&cca.atomicDestinationFilesScanned) + + if format == common.EOutputFormat.Json() { + jsonOutputTemplate := scanningProgressJsonTemplate{ + FilesScannedAtSource: srcScanned, + FilesScannedAtDestination: dstScanned, + } + outputString, err := json.Marshal(jsonOutputTemplate) + common.PanicIfErr(err) + return string(outputString) + } + + // text output + throughputString := "" + if cca.firstPartOrdered() { + throughputString = fmt.Sprintf(", 2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + } + return fmt.Sprintf("%v Files Scanned at Source, %v Files Scanned at Destination%s", + srcScanned, dstScanned, throughputString) + }) +} + +func (cca *cookedSyncCmdArgs) getJsonOfSyncJobSummary(summary common.ListSyncJobSummaryResponse) string { + // TODO figure out if deletions should be done by the enumeration engine or not + // TODO if not, remove this so that we get the proper number from the ste + summary.DeleteTotalTransfers = cca.getDeletionCount() + summary.DeleteTransfersCompleted = cca.getDeletionCount() + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) +} + func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job + var summary common.ListSyncJobSummaryResponse + var throughput float64 + var jobDone bool + + // fetch a job status and compute throughput if the first part was dispatched + if cca.firstPartOrdered() { + Rpc(common.ERpcCmd.ListSyncJobSummary(), &cca.jobID, &summary) + jobDone = summary.JobStatus.IsJobDone() + + // compute the average throughput for the last time interval + bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) * 8 / float64(1024*1024)) + timeElapsed := time.Since(cca.intervalStartTime).Seconds() + throughput = common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) + + // reset the interval timer and byte count + cca.intervalStartTime = time.Now() + cca.intervalBytesTransferred = summary.BytesOverWire + } + // first part not dispatched, and we are still scanning + // so a special message is outputted to notice the user that we are not stalling if !cca.scanningComplete() { - lcm.Progress(fmt.Sprintf("%v File Scanned at Source, %v Files Scanned at Destination", - atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned))) - return - } - // If the first part isn't ordered yet, no need to fetch the progress summary. - if !cca.firstPartOrdered() { + cca.reportScanningProgress(lcm, throughput) return } - // fetch a job status - var summary common.ListSyncJobSummaryResponse - Rpc(common.ERpcCmd.ListSyncJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() - - // if json output is desired, simply marshal and return - // note that if job is already done, we simply exit - if cca.output == common.EOutputFormat.Json() { - //jsonOutput, err := json.MarshalIndent(summary, "", " ") - jsonOutput, err := json.Marshal(summary) - common.PanicIfErr(err) - - if jobDone { - exitCode := common.EExitCode.Success() - if summary.CopyTransfersFailed+summary.DeleteTransfersFailed > 0 { - exitCode = common.EExitCode.Error() - } - lcm.Exit(string(jsonOutput), exitCode) - } else { - lcm.Info(string(jsonOutput)) - return - } - } - // if json is not desired, and job is done, then we generate a special end message to conclude the job if jobDone { - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job exitCode := common.EExitCode.Success() if summary.CopyTransfersFailed+summary.DeleteTransfersFailed > 0 { exitCode = common.EExitCode.Error() } - lcm.Exit(fmt.Sprintf( - "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Copy Transfers: %v\nTotal Number Of Delete Transfers: %v\nNumber of Copy Transfers Completed: %v\nNumber of Copy Transfers Failed: %v\nNumber of Delete Transfers Completed: %v\nNumber of Delete Transfers Failed: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - ste.ToFixed(duration.Minutes(), 4), - summary.CopyTotalTransfers, - summary.DeleteTotalTransfers, - summary.CopyTransfersCompleted, - summary.CopyTransfersFailed, - summary.DeleteTransfersCompleted, - summary.DeleteTransfersFailed, - summary.JobStatus), exitCode) - } - // if json is not needed, and job is not done, then we generate a message that goes nicely on the same line - // display a scanning keyword if the job is not completely ordered - var scanningString = "" - if !summary.CompleteJobOrdered { - scanningString = "(scanning...)" + lcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + return cca.getJsonOfSyncJobSummary(summary) + } + + return fmt.Sprintf( + ` +Job %s Summary +Files Scanned at Source: %v +Files Scanned at Destination: %v +Elapsed Time (Minutes): %v +Total Number Of Copy Transfers: %v +Number of Copy Transfers Completed: %v +Number of Copy Transfers Failed: %v +Number of Deletions at Destination: %v +Total Number of Bytes Transferred: %v +Total Number of Bytes Enumerated: %v +Final Job Status: %v +`, + summary.JobID.String(), + atomic.LoadUint64(&cca.atomicSourceFilesScanned), + atomic.LoadUint64(&cca.atomicDestinationFilesScanned), + ste.ToFixed(duration.Minutes(), 4), + summary.CopyTotalTransfers, + summary.CopyTransfersCompleted, + summary.CopyTransfersFailed, + cca.atomicDeletionCount, + summary.TotalBytesTransferred, + summary.TotalBytesEnumerated, + summary.JobStatus) + + }, exitCode) } - // compute the average throughput for the last time interval - bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) * 8 / float64(1024*1024)) - timeElapsed := time.Since(cca.intervalStartTime).Seconds() - throughPut := common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) + lcm.Progress(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + return cca.getJsonOfSyncJobSummary(summary) + } - // reset the interval timer and byte count - cca.intervalStartTime = time.Now() - cca.intervalBytesTransferred = summary.BytesOverWire + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) - lcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total%s, 2-sec Throughput (Mb/s): %v", - summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted, - summary.CopyTransfersFailed+summary.DeleteTransfersFailed, - summary.CopyTotalTransfers+summary.DeleteTotalTransfers-(summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted+summary.CopyTransfersFailed+summary.DeleteTransfersFailed), - summary.CopyTotalTransfers+summary.DeleteTotalTransfers, scanningString, ste.ToFixed(throughPut, 4))) + return fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total%s, 2-sec Throughput (Mb/s): %v%s", + summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted, + summary.CopyTransfersFailed+summary.DeleteTransfersFailed, + summary.CopyTotalTransfers+summary.DeleteTotalTransfers-(summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted+summary.CopyTransfersFailed+summary.DeleteTransfersFailed), + summary.CopyTotalTransfers+summary.DeleteTotalTransfers, perfString, ste.ToFixed(throughput, 4), diskString) + }) } func (cca *cookedSyncCmdArgs) process() (err error) { - // initialize the fields that are constant across all job part orders - jobPartOrder := common.SyncJobPartOrderRequest{ - JobID: cca.jobID, - FromTo: cca.fromTo, - LogLevel: cca.logVerbosity, - BlockSizeInBytes: cca.blockSize, - Include: cca.include, - Exclude: cca.exclude, - CommandString: cca.commandString, - SourceSAS: cca.sourceSAS, - DestinationSAS: cca.destinationSAS, - CredentialInfo: common.CredentialInfo{}, - } - ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + // verifies credential type and initializes credential info. // For sync, only one side need credential. - if jobPartOrder.CredentialInfo.CredentialType, err = getCredentialType(ctx, rawFromToInfo{ + cca.credentialInfo.CredentialType, err = getCredentialType(ctx, rawFromToInfo{ fromTo: cca.fromTo, source: cca.source, destination: cca.destination, sourceSAS: cca.sourceSAS, destinationSAS: cca.destinationSAS, - }); err != nil { + }) + + if err != nil { return err } // For OAuthToken credential, assign OAuthTokenInfo to CopyJobPartOrderRequest properly, // the info will be transferred to STE. - if jobPartOrder.CredentialInfo.CredentialType == common.ECredentialType.OAuthToken() { + if cca.credentialInfo.CredentialType == common.ECredentialType.OAuthToken() { // Message user that they are using Oauth token for authentication, // in case of silently using cached token without consciousness。 glcm.Info("Using OAuth token for authentication.") @@ -353,96 +434,34 @@ func (cca *cookedSyncCmdArgs) process() (err error) { if tokenInfo, err := uotm.GetTokenInfo(ctx); err != nil { return err } else { - jobPartOrder.CredentialInfo.OAuthTokenInfo = *tokenInfo + cca.credentialInfo.OAuthTokenInfo = *tokenInfo } } - from := cca.fromTo.From() - to := cca.fromTo.To() - switch from { - case common.ELocation.Blob(): - fromUrl, err := url.Parse(cca.source) - if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", fromUrl.String(), err.Error()) - } - blobParts := azblob.NewBlobURLParts(*fromUrl) - cca.sourceSAS = blobParts.SAS.Encode() - jobPartOrder.SourceSAS = cca.sourceSAS - blobParts.SAS = azblob.SASQueryParameters{} - bUrl := blobParts.URL() - cca.source = bUrl.String() - case common.ELocation.File(): - fromUrl, err := url.Parse(cca.source) - if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", fromUrl.String(), err.Error()) - } - fileParts := azfile.NewFileURLParts(*fromUrl) - cca.sourceSAS = fileParts.SAS.Encode() - jobPartOrder.SourceSAS = cca.sourceSAS - fileParts.SAS = azfile.SASQueryParameters{} - fUrl := fileParts.URL() - cca.source = fUrl.String() - } + var enumerator *syncEnumerator - switch to { - case common.ELocation.Blob(): - toUrl, err := url.Parse(cca.destination) + switch cca.fromTo { + case common.EFromTo.LocalBlob(): + enumerator, err = newSyncUploadEnumerator(cca) if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", toUrl.String(), err.Error()) + return err } - blobParts := azblob.NewBlobURLParts(*toUrl) - cca.destinationSAS = blobParts.SAS.Encode() - jobPartOrder.DestinationSAS = cca.destinationSAS - blobParts.SAS = azblob.SASQueryParameters{} - bUrl := blobParts.URL() - cca.destination = bUrl.String() - case common.ELocation.File(): - toUrl, err := url.Parse(cca.destination) + case common.EFromTo.BlobLocal(): + enumerator, err = newSyncDownloadEnumerator(cca) if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", toUrl.String(), err.Error()) - } - fileParts := azfile.NewFileURLParts(*toUrl) - cca.destinationSAS = fileParts.SAS.Encode() - jobPartOrder.DestinationSAS = cca.destinationSAS - fileParts.SAS = azfile.SASQueryParameters{} - fUrl := fileParts.URL() - cca.destination = fUrl.String() - } - - if from == common.ELocation.Local() { - // If the path separator is '\\', it means - // local path is a windows path - // To avoid path separator check and handling the windows - // path differently, replace the path separator with the - // the linux path separator '/' - if os.PathSeparator == '\\' { - cca.source = strings.Replace(cca.source, common.OS_PATH_SEPARATOR, "/", -1) + return err } + default: + return fmt.Errorf("the given source/destination pair is currently not supported") } - if to == common.ELocation.Local() { - // If the path separator is '\\', it means - // local path is a windows path - // To avoid path separator check and handling the windows - // path differently, replace the path separator with the - // the linux path separator '/' - if os.PathSeparator == '\\' { - cca.destination = strings.Replace(cca.destination, common.OS_PATH_SEPARATOR, "/", -1) - } - } + // trigger the progress reporting + cca.waitUntilJobCompletion(false) - switch cca.fromTo { - case common.EFromTo.LocalBlob(): - e := syncUploadEnumerator(jobPartOrder) - err = e.enumerate(cca) - case common.EFromTo.BlobLocal(): - e := syncDownloadEnumerator(jobPartOrder) - err = e.enumerate(cca) - default: - return fmt.Errorf("from to destination not supported") - } + // trigger the enumeration + err = enumerator.enumerate() if err != nil { - return fmt.Errorf("error starting the sync between source %s and destination %s. Failed with error %s", cca.source, cca.destination, err.Error()) + return err } return nil } @@ -455,6 +474,7 @@ func init() { Aliases: []string{"sc", "s"}, Short: syncCmdShortDescription, Long: syncCmdLongDescription, + Example: syncCmdExample, Args: func(cmd *cobra.Command, args []string) error { if len(args) != 2 { return fmt.Errorf("2 arguments source and destination are required for this command. Number of commands passed %d", len(args)) @@ -466,12 +486,12 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("error parsing the input given by the user. Failed with error "+err.Error(), common.EExitCode.Error()) + glcm.Error("error parsing the input given by the user. Failed with error " + err.Error()) } cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("error performing the sync between source and destination. Failed with error "+err.Error(), common.EExitCode.Error()) + glcm.Error("Cannot perform sync due to error: " + err.Error()) } glcm.SurrenderControl() @@ -479,17 +499,18 @@ func init() { } rootCmd.AddCommand(syncCmd) - syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when syncing destination to source.") - syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "Use this block size when source to Azure Storage or from Azure Storage.") - // hidden filters - syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "Filter: only include these files when copying. "+ - "Support use of *. More than one file are separated by ';'") - syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "Filter: Follow symbolic links when performing sync from local file system.") - syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "Filter: Exclude these files when copying. Support use of *.") - syncCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") - syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") - syncCmd.PersistentFlags().BoolVar(&raw.force, "force", false, "defines user's decision to delete extra files at the destination that are not present at the source. "+ - "If false, user will be prompted with a question while scheduling files/blobs for deletion.") + syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", true, "true by default, look into sub-directories recursively when syncing between directories.") + syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") + syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") + syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") + syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") + syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+ + "If set to prompt, user will be asked a question before scheduling files/blobs for deletion.") + syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") + // TODO: should the previous line list the allowable values? + + // TODO follow sym link is not implemented, clarify behavior first + //syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when performing sync from local file system.") // TODO sync does not support any BlobAttributes, this functionality should be added } diff --git a/cmd/syncComparator.go b/cmd/syncComparator.go new file mode 100644 index 000000000..173043046 --- /dev/null +++ b/cmd/syncComparator.go @@ -0,0 +1,106 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +// with the help of an objectIndexer containing the source objects +// find out the destination objects that should be transferred +// in other words, this should be used when destination is being enumerated secondly +type syncDestinationComparator struct { + // the rejected objects would be passed to the destinationCleaner + destinationCleaner objectProcessor + + // the processor responsible for scheduling copy transfers + copyTransferScheduler objectProcessor + + // storing the source objects + sourceIndex *objectIndexer +} + +func newSyncDestinationComparator(i *objectIndexer, copyScheduler, cleaner objectProcessor) *syncDestinationComparator { + return &syncDestinationComparator{sourceIndex: i, copyTransferScheduler: copyScheduler, destinationCleaner: cleaner} +} + +// it will only schedule transfers for destination objects that are present in the indexer but stale compared to the entry in the map +// if the destinationObject is not at the source, it will be passed to the destinationCleaner +// ex: we already know what the source contains, now we are looking at objects at the destination +// if file x from the destination exists at the source, then we'd only transfer it if it is considered stale compared to its counterpart at the source +// if file x does not exist at the source, then it is considered extra, and will be deleted +func (f *syncDestinationComparator) processIfNecessary(destinationObject storedObject) error { + sourceObjectInMap, present := f.sourceIndex.indexMap[destinationObject.relativePath] + + // if the destinationObject is present at source and stale, we transfer the up-to-date version from source + if present { + defer delete(f.sourceIndex.indexMap, destinationObject.relativePath) + + if sourceObjectInMap.isMoreRecentThan(destinationObject) { + err := f.copyTransferScheduler(sourceObjectInMap) + if err != nil { + return err + } + } + } else { + // purposefully ignore the error from destinationCleaner + // it's a tolerable error, since it just means some extra destination object might hang around a bit longer + _ = f.destinationCleaner(destinationObject) + } + + return nil +} + +// with the help of an objectIndexer containing the destination objects +// filter out the source objects that should be transferred +// in other words, this should be used when source is being enumerated secondly +type syncSourceComparator struct { + // the processor responsible for scheduling copy transfers + copyTransferScheduler objectProcessor + + // storing the destination objects + destinationIndex *objectIndexer +} + +func newSyncSourceComparator(i *objectIndexer, copyScheduler objectProcessor) *syncSourceComparator { + return &syncSourceComparator{destinationIndex: i, copyTransferScheduler: copyScheduler} +} + +// it will only transfer source items that are: +// 1. not present in the map +// 2. present but is more recent than the entry in the map +// note: we remove the storedObject if it is present so that when we have finished +// the index will contain all objects which exist at the destination but were NOT seen at the source +func (f *syncSourceComparator) processIfNecessary(sourceObject storedObject) error { + destinationObjectInMap, present := f.destinationIndex.indexMap[sourceObject.relativePath] + + if present { + defer delete(f.destinationIndex.indexMap, sourceObject.relativePath) + + // if destination is stale, schedule source for transfer + if sourceObject.isMoreRecentThan(destinationObjectInMap) { + return f.copyTransferScheduler(sourceObject) + + } else { + // skip if source is more recent + return nil + } + } + + // if source does not exist at the destination, then schedule it for transfer + return f.copyTransferScheduler(sourceObject) +} diff --git a/cmd/syncDownloadEnumerator.go b/cmd/syncDownloadEnumerator.go deleted file mode 100644 index fafe2280c..000000000 --- a/cmd/syncDownloadEnumerator.go +++ /dev/null @@ -1,481 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - - "sync/atomic" - - "time" - - "github.com/Azure/azure-pipeline-go/pipeline" - "github.com/Azure/azure-storage-azcopy/common" - "github.com/Azure/azure-storage-azcopy/ste" - "github.com/Azure/azure-storage-blob-go/azblob" -) - -type syncDownloadEnumerator common.SyncJobPartOrderRequest - -// accept a new transfer, if the threshold is reached, dispatch a job part order -func (e *syncDownloadEnumerator) addTransferToUpload(transfer common.CopyTransfer, cca *cookedSyncCmdArgs) error { - - if len(e.CopyJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { - resp := common.CopyJobPartOrderResponse{} - e.CopyJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // if the current part order sent to engine is 0, then set atomicSyncStatus - // variable to 1 - if e.PartNumber == 0 { - //cca.waitUntilJobCompletion(false) - cca.setFirstPartOrdered() - } - e.CopyJobRequest.Transfers = []common.CopyTransfer{} - e.PartNumber++ - } - e.CopyJobRequest.Transfers = append(e.CopyJobRequest.Transfers, transfer) - return nil -} - -// addTransferToDelete adds the filePath to the list of files to delete locally. -func (e *syncDownloadEnumerator) addTransferToDelete(filePath string) { - e.FilesToDeleteLocally = append(e.FilesToDeleteLocally, filePath) -} - -// we need to send a last part with isFinalPart set to true, along with whatever transfers that still haven't been sent -func (e *syncDownloadEnumerator) dispatchFinalPart(cca *cookedSyncCmdArgs) error { - numberOfCopyTransfers := len(e.CopyJobRequest.Transfers) - numberOfDeleteTransfers := len(e.FilesToDeleteLocally) - // If the numberoftransfer to copy / delete both are 0 - // means no transfer has been to queue to send to STE - if numberOfCopyTransfers == 0 && numberOfDeleteTransfers == 0 { - glcm.Exit("cannot start job because there are no transfer to upload or delete. "+ - "The source and destination are in sync", 0) - return nil - } - if numberOfCopyTransfers > 0 { - // Only CopyJobPart Order needs to be sent - e.CopyJobRequest.IsFinalPart = true - e.CopyJobRequest.PartNum = e.PartNumber - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // If the JobPart sent was the first part, then set atomicSyncStatus to 1, so that progress reporting can start. - if e.PartNumber == 0 { - cca.setFirstPartOrdered() - } - } - if numberOfDeleteTransfers > 0 { - answer := "" - if cca.force { - answer = "y" - } else { - answer = glcm.Prompt(fmt.Sprintf("Sync has enumerated %v files to delete locally. Do you want to delete these files ? Please confirm with y/n: ", numberOfDeleteTransfers)) - } - // read a line from stdin, if the answer is not yes, then is No, then ignore the transfers queued for deletion and continue - if !strings.EqualFold(answer, "y") { - if numberOfCopyTransfers == 0 { - glcm.Exit("cannot start job because there are no transfer to upload or delete. "+ - "The source and destination are in sync", 0) - } - cca.isEnumerationComplete = true - return nil - } - for _, file := range e.FilesToDeleteLocally { - err := os.Remove(file) - if err != nil { - glcm.Info(fmt.Sprintf("error %s deleting the file %s", err.Error(), file)) - } - } - if numberOfCopyTransfers == 0 { - glcm.Exit(fmt.Sprintf("sync completed. Deleted %v files locally ", len(e.FilesToDeleteLocally)), 0) - } - } - cca.isEnumerationComplete = true - return nil -} - -// listDestinationAndCompare lists the blob under the destination mentioned and verifies whether the blob -// exists locally or not by checking the expected localPath of blob in the sourceFiles map. If the blob -// does exists, it compares the last modified time. If it does not exists, it queues the blob for deletion. -func (e *syncDownloadEnumerator) listSourceAndCompare(cca *cookedSyncCmdArgs, p pipeline.Pipeline) error { - util := copyHandlerUtil{} - - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - // rootPath is the path of destination without wildCards - // For Example: cca.source = C:\a1\a* , so rootPath = C:\a1 - rootPath, _ := util.sourceRootPathWithoutWildCards(cca.destination) - //replace the os path separator with path separator "/" which is path separator for blobs - //sourcePattern = strings.Replace(sourcePattern, string(os.PathSeparator), "/", -1) - sourceURL, err := url.Parse(cca.source) - if err != nil { - return fmt.Errorf("error parsing the destinatio url") - } - - // since source is a remote url, it will have sas parameter - // since sas parameter will be stripped from the source url - // while cooking the raw command arguments - // source sas is added to url for listing the blobs. - sourceURL = util.appendQueryParamToUrl(sourceURL, cca.sourceSAS) - - blobUrlParts := azblob.NewBlobURLParts(*sourceURL) - blobURLPartsExtension := blobURLPartsExtension{blobUrlParts} - - containerUrl := util.getContainerUrl(blobUrlParts) - searchPrefix, pattern, _ := blobURLPartsExtension.searchPrefixFromBlobURL() - - containerBlobUrl := azblob.NewContainerURL(containerUrl, p) - - // virtual directory is the entire virtual directory path before the blob name - // passed in the searchPrefix - // Example: cca.destination = https:///vd-1? searchPrefix = vd-1/ - // virtualDirectory = vd-1 - // Example: cca.destination = https:///vd-1/vd-2/fi*.txt? searchPrefix = vd-1/vd-2/fi*.txt - // virtualDirectory = vd-1/vd-2/ - virtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) - // strip away the leading / in the closest virtual directory - if len(virtualDirectory) > 0 && virtualDirectory[0:1] == "/" { - virtualDirectory = virtualDirectory[1:] - } - - // Get the destination path without the wildcards - // This is defined since the files mentioned with exclude flag - // & include flag are relative to the Destination - // If the Destination has wildcards, then files are relative to the - // parent Destination path which is the path of last directory in the Destination - // without wildcards - // For Example: dst = "/home/user/dir1" parentSourcePath = "/home/user/dir1" - // For Example: dst = "/home/user/dir*" parentSourcePath = "/home/user" - // For Example: dst = "/home/*" parentSourcePath = "/home" - parentSourcePath := blobUrlParts.BlobName - wcIndex := util.firstIndexOfWildCard(parentSourcePath) - if wcIndex != -1 { - parentSourcePath = parentSourcePath[:wcIndex] - pathSepIndex := strings.LastIndex(parentSourcePath, common.AZCOPY_PATH_SEPARATOR_STRING) - parentSourcePath = parentSourcePath[:pathSepIndex] - } - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerBlobUrl.ListBlobsFlatSegment(ctx, marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Segment.BlobItems { - // check if the listed blob segment does not matches the sourcePath pattern - // if it does not comparison is not required - if !util.matchBlobNameAgainstPattern(pattern, blobInfo.Name, cca.recursive) { - continue - } - // realtivePathofBlobLocally is the local path relative to source at which blob should be downloaded - // Example: cca.source ="C:\User1\user-1" cca.destination = "https:///virtual-dir?" blob name = "virtual-dir/a.txt" - // realtivePathofBlobLocally = virtual-dir/a.txt - relativePathofBlobLocally := util.relativePathToRoot(parentSourcePath, blobInfo.Name, '/') - relativePathofBlobLocally = strings.Replace(relativePathofBlobLocally, virtualDirectory, "", 1) - - blobLocalPath := util.generateLocalPath(cca.destination, relativePathofBlobLocally) - - // Increment the number of files scanned at the destination. - atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) - - // calculate the expected local path of the blob - blobLocalPath = util.generateLocalPath(rootPath, relativePathofBlobLocally) - - // If the files is found in the list of files to be excluded, then it is not compared - _, found := e.SourceFilesToExclude[blobLocalPath] - if found { - continue - } - // Check if the blob exists in the map of source Files. If the file is - // found, compare the modified time of the file against the blob's last - // modified time. If the modified time of file is later than the blob's - // modified time, then queue transfer for upload. If not, then delete - // blobLocalPath from the map of sourceFiles. - localFileTime, found := e.SourceFiles[blobLocalPath] - if found { - if !blobInfo.Properties.LastModified.After(localFileTime) { - delete(e.SourceFiles, blobLocalPath) - continue - } - } - e.addTransferToUpload(common.CopyTransfer{ - Source: util.stripSASFromBlobUrl(util.generateBlobUrl(containerUrl, blobInfo.Name)).String(), - Destination: blobLocalPath, - SourceSize: *blobInfo.Properties.ContentLength, - LastModifiedTime: blobInfo.Properties.LastModified, - }, cca) - - delete(e.SourceFiles, blobLocalPath) - } - marker = listBlob.NextMarker - } - return nil -} - -func (e *syncDownloadEnumerator) listTheDestinationIfRequired(cca *cookedSyncCmdArgs, p pipeline.Pipeline) (bool, error) { - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - util := copyHandlerUtil{} - - // attempt to parse the destination url - sourceURL, err := url.Parse(cca.source) - // the destination should have already been validated, it would be surprising if it cannot be parsed at this point - common.PanicIfErr(err) - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - sourceURL = util.appendQueryParamToUrl(sourceURL, cca.sourceSAS) - - blobUrl := azblob.NewBlobURL(*sourceURL, p) - - // Get the files and directories for the given source pattern - listOfFilesAndDir, lofaderr := filepath.Glob(cca.destination) - if lofaderr != nil { - return false, fmt.Errorf("error getting the files and directories for source pattern %s", cca.source) - } - - // Get the blob Properties - bProperties, bPropertiesError := blobUrl.GetProperties(ctx, azblob.BlobAccessConditions{}) - - // isSourceASingleFile is used to determine whether given source pattern represents single file or not - // If the source is a single file, this pointer will not be nil - // if it is nil, it means the source is a directory or list of file - var isSourceASingleFile os.FileInfo = nil - - if len(listOfFilesAndDir) == 0 { - fInfo, fError := os.Stat(listOfFilesAndDir[0]) - if fError != nil { - return false, fmt.Errorf("cannot get the information of the %s. Failed with error %s", listOfFilesAndDir[0], fError) - } - if fInfo.Mode().IsRegular() { - isSourceASingleFile = fInfo - } - } - - // sync only happens between the source and destination of same type i.e between blob and blob or between Directory and Virtual Folder / Container - // If the source is a file and destination is not a blob, sync fails. - if isSourceASingleFile != nil && bPropertiesError != nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between file %s and non blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - // If the source is a directory and destination is a blob - if isSourceASingleFile == nil && bPropertiesError == nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between directory %s and blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - - // If both source is a file and destination is a blob, then we need to do the comparison and queue the transfer if required. - if isSourceASingleFile != nil && bPropertiesError == nil { - blobName := sourceURL.Path[strings.LastIndex(sourceURL.Path, "/")+1:] - // Compare the blob name and file name - // blobName and filename should be same for sync to happen - if strings.Compare(blobName, isSourceASingleFile.Name()) != 0 { - glcm.Exit(fmt.Sprintf("sync cannot be done since blob %s and filename %s doesn't match", blobName, isSourceASingleFile.Name()), 1) - } - - // If the modified time of file local is not later than that of blob - // sync does not needs to happen. - if isSourceASingleFile.ModTime().After(bProperties.LastModified()) { - glcm.Exit(fmt.Sprintf("blob %s and file %s already in sync", blobName, isSourceASingleFile.Name()), 1) - } - - e.addTransferToUpload(common.CopyTransfer{ - Source: util.stripSASFromBlobUrl(*sourceURL).String(), - Destination: cca.source, - SourceSize: bProperties.ContentLength(), - LastModifiedTime: bProperties.LastModified(), - }, cca) - } - - sourcePattern := "" - // Parse the source URL into blob URL parts. - blobUrlParts := azblob.NewBlobURLParts(*sourceURL) - // get the root path without wildCards and get the source Pattern - // For Example: source = /a*/*/* - // rootPath = sourcePattern = a*/*/* - blobUrlParts.BlobName, sourcePattern = util.sourceRootPathWithoutWildCards(blobUrlParts.BlobName) - - // Iterate through each file / dir inside the source - // and then checkAndQueue - for _, fileOrDir := range listOfFilesAndDir { - f, err := os.Stat(fileOrDir) - if err != nil { - glcm.Info(fmt.Sprintf("cannot get the file info for %s. failed with error %s", fileOrDir, err.Error())) - } - // directories are uploaded only if recursive is on - if f.IsDir() && cca.recursive { - // walk goes through the entire directory tree - err = filepath.Walk(fileOrDir, func(pathToFile string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fileInfo.IsDir() { - return nil - } else { - // replace the OS path separator in pathToFile string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - pathToFile = strings.Replace(pathToFile, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - // localfileRelativePath is the path of file relative to root directory - // Example1: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\a.txt localfileRelativePath = \a.txt - // Example2: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - localfileRelativePath := strings.Replace(pathToFile, cca.destination, "", 1) - // remove the path separator at the start of relative path - if len(localfileRelativePath) > 0 && localfileRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { - localfileRelativePath = localfileRelativePath[1:] - } - // if the localfileRelativePath does not match the source pattern, then it is not compared - if !util.matchBlobNameAgainstPattern(sourcePattern, localfileRelativePath, cca.recursive) { - return nil - } - - if util.resourceShouldBeExcluded(cca.destination, e.Exclude, pathToFile) { - e.SourceFilesToExclude[pathToFile] = fileInfo.ModTime() - return nil - } - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[pathToFile] = fileInfo.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) - } - return nil - }) - } else if !f.IsDir() { - // replace the OS path separator in fileOrDir string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - fileOrDir = strings.Replace(fileOrDir, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - // localfileRelativePath is the path of file relative to root directory - // Example1: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\a.txt localfileRelativePath = \a.txt - // Example2: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - localfileRelativePath := strings.Replace(fileOrDir, cca.destination, "", 1) - // remove the path separator at the start of relative path - if len(localfileRelativePath) > 0 && localfileRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { - localfileRelativePath = localfileRelativePath[1:] - } - // if the localfileRelativePath does not match the source pattern, then it is not compared - if !util.matchBlobNameAgainstPattern(sourcePattern, localfileRelativePath, cca.recursive) { - continue - } - - if util.resourceShouldBeExcluded(cca.destination, e.Exclude, fileOrDir) { - e.SourceFilesToExclude[fileOrDir] = f.ModTime() - continue - } - - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[fileOrDir] = f.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) - } - } - return false, nil -} - -// queueSourceFilesForUpload -func (e *syncDownloadEnumerator) queueSourceFilesForUpload(cca *cookedSyncCmdArgs) { - for file, _ := range e.SourceFiles { - e.addTransferToDelete(file) - } -} - -// this function accepts the list of files/directories to transfer and processes them -func (e *syncDownloadEnumerator) enumerate(cca *cookedSyncCmdArgs) error { - ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - p, err := createBlobPipeline(ctx, e.CredentialInfo) - if err != nil { - return err - } - - // Copying the JobId of sync job to individual copyJobRequest - e.CopyJobRequest.JobID = e.JobID - // Copying the FromTo of sync job to individual copyJobRequest - e.CopyJobRequest.FromTo = e.FromTo - - // set the sas of user given Source - e.CopyJobRequest.SourceSAS = e.SourceSAS - - // set the sas of user given destination - e.CopyJobRequest.DestinationSAS = e.DestinationSAS - - // Set the preserve-last-modified-time to true in CopyJobRequest - e.CopyJobRequest.BlobAttributes.PreserveLastModifiedTime = true - - // Copying the JobId of sync job to individual deleteJobRequest. - e.DeleteJobRequest.JobID = e.JobID - // FromTo of DeleteJobRequest will be BlobTrash. - e.DeleteJobRequest.FromTo = common.EFromTo.BlobTrash() - - // set the sas of user given Source - e.DeleteJobRequest.SourceSAS = e.SourceSAS - - // set the sas of user given destination - e.DeleteJobRequest.DestinationSAS = e.DestinationSAS - - // set force wriet flag to true - e.CopyJobRequest.ForceWrite = true - - //Set the log level - e.CopyJobRequest.LogLevel = e.LogLevel - e.DeleteJobRequest.LogLevel = e.LogLevel - - // Copy the sync Command String to the CopyJobPartRequest and DeleteJobRequest - e.CopyJobRequest.CommandString = e.CommandString - e.DeleteJobRequest.CommandString = e.CommandString - - // Set credential info properly - e.CopyJobRequest.CredentialInfo = e.CredentialInfo - e.DeleteJobRequest.CredentialInfo = e.CredentialInfo - - e.SourceFiles = make(map[string]time.Time) - - e.SourceFilesToExclude = make(map[string]time.Time) - - cca.waitUntilJobCompletion(false) - - isSourceABlob, err := e.listTheDestinationIfRequired(cca, p) - if err != nil { - return err - } - - // If the source provided is a blob, then remote doesn't needs to be compared against the local - // since single blob already has been compared against the file - if !isSourceABlob { - err = e.listSourceAndCompare(cca, p) - if err != nil { - return err - } - } - - e.queueSourceFilesForUpload(cca) - - // No Job Part has been dispatched, then dispatch the JobPart. - if e.PartNumber == 0 || - len(e.CopyJobRequest.Transfers) > 0 || - len(e.DeleteJobRequest.Transfers) > 0 { - err = e.dispatchFinalPart(cca) - if err != nil { - return err - } - cca.setFirstPartOrdered() - } - // scanning all the destination and is complete - cca.setScanningComplete() - return nil -} diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go new file mode 100644 index 000000000..810a3ad50 --- /dev/null +++ b/cmd/syncEnumerator.go @@ -0,0 +1,164 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "errors" + "fmt" + "github.com/Azure/azure-storage-azcopy/common" +) + +// -------------------------------------- Implemented Enumerators -------------------------------------- \\ + +// download implies transferring from a remote resource to the local disk +// in this scenario, the destination is scanned/indexed first +// then the source is scanned and filtered based on what the destination contains +// we do the local one first because it is assumed that local file systems will be faster to enumerate than remote resources +func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { + destinationTraverser, err := newLocalTraverserForSync(cca, false) + if err != nil { + return nil, err + } + + sourceTraverser, err := newBlobTraverserForSync(cca, true) + if err != nil { + return nil, err + } + + // verify that the traversers are targeting the same type of resources + _, isSingleBlob := sourceTraverser.getPropertiesIfSingleBlob() + _, isSingleFile, _ := destinationTraverser.getInfoIfSingleFile() + if isSingleBlob != isSingleFile { + return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") + } + + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart, isSingleBlob && isSingleFile) + includeFilters := buildIncludeFilters(cca.include) + excludeFilters := buildExcludeFilters(cca.exclude) + + // set up the filters in the right order + filters := append(includeFilters, excludeFilters...) + + // set up the comparator so that the source/destination can be compared + indexer := newObjectIndexer() + comparator := newSyncSourceComparator(indexer, transferScheduler.scheduleCopyTransfer) + + finalize := func() error { + // remove the extra files at the destination that were not present at the source + // we can only know what needs to be deleted when we have FINISHED traversing the remote source + // since only then can we know which local files definitely don't exist remotely + deleteScheduler := newSyncLocalDeleteProcessor(cca) + err = indexer.traverse(deleteScheduler.removeImmediately, nil) + if err != nil { + return err + } + + // let the deletions happen first + // otherwise if the final part is executed too quickly, we might quit before deletions could finish + jobInitiated, err := transferScheduler.dispatchFinalPart() + if err != nil { + return err + } + + quitIfInSync(jobInitiated, cca.getDeletionCount() > 0, cca) + cca.setScanningComplete() + return nil + } + + return newSyncEnumerator(destinationTraverser, sourceTraverser, indexer, filters, + comparator.processIfNecessary, finalize), nil +} + +// upload implies transferring from a local disk to a remote resource +// in this scenario, the local disk (source) is scanned/indexed first +// then the destination is scanned and filtered based on what the destination contains +// we do the local one first because it is assumed that local file systems will be faster to enumerate than remote resources +func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { + sourceTraverser, err := newLocalTraverserForSync(cca, true) + if err != nil { + return nil, err + } + + destinationTraverser, err := newBlobTraverserForSync(cca, false) + if err != nil { + return nil, err + } + + // verify that the traversers are targeting the same type of resources + _, isSingleBlob := destinationTraverser.getPropertiesIfSingleBlob() + _, isSingleFile, _ := sourceTraverser.getInfoIfSingleFile() + if isSingleBlob != isSingleFile { + return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") + } + + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart, isSingleBlob && isSingleFile) + includeFilters := buildIncludeFilters(cca.include) + excludeFilters := buildExcludeFilters(cca.exclude) + + // set up the filters in the right order + filters := append(includeFilters, excludeFilters...) + + // set up the comparator so that the source/destination can be compared + indexer := newObjectIndexer() + destinationCleaner, err := newSyncBlobDeleteProcessor(cca) + if err != nil { + return nil, fmt.Errorf("unable to instantiate destination cleaner due to: %s", err.Error()) + } + // when uploading, we can delete remote objects immediately, because as we traverse the remote location + // we ALREADY have available a complete map of everything that exists locally + // so as soon as we see a remote destination object we can know whether it exists in the local source + comparator := newSyncDestinationComparator(indexer, transferScheduler.scheduleCopyTransfer, destinationCleaner.removeImmediately) + + finalize := func() error { + // schedule every local file that doesn't exist at the destination + err = indexer.traverse(transferScheduler.scheduleCopyTransfer, filters) + if err != nil { + return err + } + + jobInitiated, err := transferScheduler.dispatchFinalPart() + if err != nil { + return err + } + + quitIfInSync(jobInitiated, cca.getDeletionCount() > 0, cca) + cca.setScanningComplete() + return nil + } + + return newSyncEnumerator(sourceTraverser, destinationTraverser, indexer, filters, + comparator.processIfNecessary, finalize), nil +} + +func quitIfInSync(transferJobInitiated, anyDestinationFileDeleted bool, cca *cookedSyncCmdArgs) { + if !transferJobInitiated && !anyDestinationFileDeleted { + cca.reportScanningProgress(glcm, 0) + glcm.Exit(func(format common.OutputFormat) string { + return "The source and destination are already in sync." + }, common.EExitCode.Success()) + } else if !transferJobInitiated && anyDestinationFileDeleted { + // some files were deleted but no transfer scheduled + cca.reportScanningProgress(glcm, 0) + glcm.Exit(func(format common.OutputFormat) string { + return "The source and destination are now in sync." + }, common.EExitCode.Success()) + } +} diff --git a/cmd/syncIndexer.go b/cmd/syncIndexer.go new file mode 100644 index 000000000..5d1e8313f --- /dev/null +++ b/cmd/syncIndexer.go @@ -0,0 +1,60 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "fmt" +) + +// the objectIndexer is essential for the generic sync enumerator to work +// it can serve as a: +// 1. objectProcessor: accumulate a lookup map with given storedObjects +// 2. resourceTraverser: go through the entities in the map like a traverser +type objectIndexer struct { + indexMap map[string]storedObject + counter int +} + +func newObjectIndexer() *objectIndexer { + return &objectIndexer{indexMap: make(map[string]storedObject)} +} + +// process the given stored object by indexing it using its relative path +func (i *objectIndexer) store(storedObject storedObject) (err error) { + if i.counter == MaxNumberOfFilesAllowedInSync { + return fmt.Errorf("the maxium number of file allowed in sync is: %v", MaxNumberOfFilesAllowedInSync) + } + + i.indexMap[storedObject.relativePath] = storedObject + i.counter += 1 + return +} + +// go through the remaining stored objects in the map to process them +func (i *objectIndexer) traverse(processor objectProcessor, filters []objectFilter) (err error) { + for _, value := range i.indexMap { + err = processIfPassedFilters(filters, value, processor) + if err != nil { + return + } + } + return +} diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go new file mode 100644 index 000000000..5f9dc8cce --- /dev/null +++ b/cmd/syncProcessor.go @@ -0,0 +1,204 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "fmt" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-azcopy/ste" + "github.com/Azure/azure-storage-blob-go/azblob" + "net/url" + "os" + "path" + "path/filepath" +) + +// extract the right info from cooked arguments and instantiate a generic copy transfer processor from it +func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int, isSingleFileSync bool) *copyTransferProcessor { + copyJobTemplate := &common.CopyJobPartOrderRequest{ + JobID: cca.jobID, + CommandString: cca.commandString, + FromTo: cca.fromTo, + SourceRoot: replacePathSeparators(cca.source), + DestinationRoot: replacePathSeparators(cca.destination), + + // authentication related + CredentialInfo: cca.credentialInfo, + SourceSAS: cca.sourceSAS, + DestinationSAS: cca.destinationSAS, + + // flags + BlobAttributes: common.BlobTransferAttributes{ + PreserveLastModifiedTime: true, // must be true for sync so that future syncs have this information available + MD5ValidationOption: cca.md5ValidationOption, + BlockSizeInBytes: cca.blockSize}, + ForceWrite: true, // once we decide to transfer for a sync operation, we overwrite the destination regardless + LogLevel: cca.logVerbosity, + } + + if !isSingleFileSync { + copyJobTemplate.SourceRoot += common.AZCOPY_PATH_SEPARATOR_STRING + copyJobTemplate.DestinationRoot += common.AZCOPY_PATH_SEPARATOR_STRING + } + + reportFirstPart := func() { cca.setFirstPartOrdered() } + reportFinalPart := func() { cca.isEnumerationComplete = true } + + shouldEncodeSource := cca.fromTo.From().IsRemote() + shouldEncodeDestination := cca.fromTo.To().IsRemote() + + // note that the source and destination, along with the template are given to the generic processor's constructor + // this means that given an object with a relative path, this processor already knows how to schedule the right kind of transfers + return newCopyTransferProcessor(copyJobTemplate, numOfTransfersPerPart, cca.source, cca.destination, + shouldEncodeSource, shouldEncodeDestination, reportFirstPart, reportFinalPart) +} + +// base for delete processors targeting different resources +type interactiveDeleteProcessor struct { + // the plugged-in deleter that performs the actual deletion + deleter objectProcessor + + // whether we should ask the user for permission the first time we delete a file + shouldPromptUser bool + + // note down whether any delete should happen + shouldDelete bool + + // used for prompt message + // examples: "blobs", "local files", etc. + objectTypeToDisplay string + + // used for prompt message + // examples: a directory path, or url to container + objectLocationToDisplay string + + // count the deletions that happened + incrementDeletionCount func() +} + +func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err error) { + if d.shouldPromptUser { + d.shouldDelete = d.promptForConfirmation() // note down the user's decision + d.shouldPromptUser = false // only prompt the first time that this function is called + } + + if !d.shouldDelete { + return nil + } + + err = d.deleter(object) + if err != nil { + glcm.Info(fmt.Sprintf("error %s deleting the object %s", err.Error(), object.relativePath)) + } + + if d.incrementDeletionCount != nil { + d.incrementDeletionCount() + } + return +} + +func (d *interactiveDeleteProcessor) promptForConfirmation() (shouldDelete bool) { + shouldDelete = false + + answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them from the destination(%s)? Please confirm with y/n (default: n): ", + d.objectTypeToDisplay, d.objectLocationToDisplay)) + if answer == "y" || answer == "yes" { + shouldDelete = true + glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectTypeToDisplay)) + } else { + glcm.Info("No deletions will happen.") + } + return +} + +func newInteractiveDeleteProcessor(deleter objectProcessor, deleteDestination common.DeleteDestination, + objectTypeToDisplay string, objectLocationToDisplay string, incrementDeletionCounter func()) *interactiveDeleteProcessor { + + return &interactiveDeleteProcessor{ + deleter: deleter, + objectTypeToDisplay: objectTypeToDisplay, + objectLocationToDisplay: objectLocationToDisplay, + incrementDeletionCount: incrementDeletionCounter, + shouldPromptUser: deleteDestination == common.EDeleteDestination.Prompt(), + shouldDelete: deleteDestination == common.EDeleteDestination.True(), // if shouldPromptUser is true, this will start as false, but we will determine its value later + } +} + +func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs) *interactiveDeleteProcessor { + localDeleter := localFileDeleter{rootPath: cca.destination} + return newInteractiveDeleteProcessor(localDeleter.deleteFile, cca.deleteDestination, "local files", cca.destination, cca.incrementDeletionCount) +} + +type localFileDeleter struct { + rootPath string +} + +func (l *localFileDeleter) deleteFile(object storedObject) error { + glcm.Info("Deleting extra file: " + object.relativePath) + return os.Remove(filepath.Join(l.rootPath, object.relativePath)) +} + +func newSyncBlobDeleteProcessor(cca *cookedSyncCmdArgs) (*interactiveDeleteProcessor, error) { + rawURL, err := url.Parse(cca.destination) + if err != nil { + return nil, err + } else if err == nil && cca.destinationSAS != "" { + copyHandlerUtil{}.appendQueryParamToUrl(rawURL, cca.destinationSAS) + } + + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p, err := createBlobPipeline(ctx, cca.credentialInfo) + if err != nil { + return nil, err + } + + return newInteractiveDeleteProcessor(newBlobDeleter(rawURL, p, ctx).deleteBlob, + cca.deleteDestination, "blobs", cca.destination, cca.incrementDeletionCount), nil +} + +type blobDeleter struct { + rootURL *url.URL + p pipeline.Pipeline + ctx context.Context +} + +func newBlobDeleter(rawRootURL *url.URL, p pipeline.Pipeline, ctx context.Context) *blobDeleter { + return &blobDeleter{ + rootURL: rawRootURL, + p: p, + ctx: ctx, + } +} + +func (b *blobDeleter) deleteBlob(object storedObject) error { + glcm.Info("Deleting extra blob: " + object.relativePath) + + // construct the blob URL using its relative path + // the rootURL could be pointing to a container, or a virtual directory + blobURLParts := azblob.NewBlobURLParts(*b.rootURL) + blobURLParts.BlobName = path.Join(blobURLParts.BlobName, object.relativePath) + + blobURL := azblob.NewBlobURL(blobURLParts.URL(), b.p) + _, err := blobURL.Delete(b.ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) + return err +} diff --git a/cmd/syncTraverser.go b/cmd/syncTraverser.go new file mode 100644 index 000000000..dd8bc8dd6 --- /dev/null +++ b/cmd/syncTraverser.go @@ -0,0 +1,106 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "errors" + "github.com/Azure/azure-storage-azcopy/ste" + "net/url" + "path" + "strings" + "sync/atomic" +) + +func newLocalTraverserForSync(cca *cookedSyncCmdArgs, isSource bool) (*localTraverser, error) { + var fullPath string + + if isSource { + fullPath = path.Clean(cca.source) + } else { + fullPath = path.Clean(cca.destination) + } + + if strings.ContainsAny(fullPath, "*?") { + return nil, errors.New("illegal local path, no pattern matching allowed for sync command") + } + + incrementEnumerationCounter := func() { + var counterAddr *uint64 + + if isSource { + counterAddr = &cca.atomicSourceFilesScanned + } else { + counterAddr = &cca.atomicDestinationFilesScanned + } + + atomic.AddUint64(counterAddr, 1) + } + + traverser := newLocalTraverser(fullPath, cca.recursive, incrementEnumerationCounter) + + return traverser, nil +} + +func newBlobTraverserForSync(cca *cookedSyncCmdArgs, isSource bool) (t *blobTraverser, err error) { + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + + // figure out the right URL + var rawURL *url.URL + if isSource { + rawURL, err = url.Parse(cca.source) + if err == nil && cca.sourceSAS != "" { + copyHandlerUtil{}.appendQueryParamToUrl(rawURL, cca.sourceSAS) + } + } else { + rawURL, err = url.Parse(cca.destination) + if err == nil && cca.destinationSAS != "" { + copyHandlerUtil{}.appendQueryParamToUrl(rawURL, cca.destinationSAS) + } + } + + if err != nil { + return + } + + if strings.Contains(rawURL.Path, "*") { + return nil, errors.New("illegal URL, no pattern matching allowed for sync command") + } + + p, err := createBlobPipeline(ctx, cca.credentialInfo) + if err != nil { + return + } + + incrementEnumerationCounter := func() { + var counterAddr *uint64 + + if isSource { + counterAddr = &cca.atomicSourceFilesScanned + } else { + counterAddr = &cca.atomicDestinationFilesScanned + } + + atomic.AddUint64(counterAddr, 1) + } + + return newBlobTraverser(rawURL, p, ctx, cca.recursive, incrementEnumerationCounter), nil +} diff --git a/cmd/syncUploadEnumerator.go b/cmd/syncUploadEnumerator.go deleted file mode 100644 index 249cc155e..000000000 --- a/cmd/syncUploadEnumerator.go +++ /dev/null @@ -1,563 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - "sync/atomic" - "time" - - "github.com/Azure/azure-pipeline-go/pipeline" - "github.com/Azure/azure-storage-azcopy/common" - "github.com/Azure/azure-storage-azcopy/ste" - "github.com/Azure/azure-storage-blob-go/azblob" -) - -type syncUploadEnumerator common.SyncJobPartOrderRequest - -// accepts a new transfer which is to delete the blob on container. -func (e *syncUploadEnumerator) addTransferToDelete(transfer common.CopyTransfer, cca *cookedSyncCmdArgs) error { - e.DeleteJobRequest.Transfers = append(e.DeleteJobRequest.Transfers, transfer) - return nil -} - -// accept a new transfer, if the threshold is reached, dispatch a job part order -func (e *syncUploadEnumerator) addTransferToUpload(transfer common.CopyTransfer, cca *cookedSyncCmdArgs) error { - - if len(e.CopyJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { - resp := common.CopyJobPartOrderResponse{} - e.CopyJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // if the current part order sent to engine is 0, then start fetching the Job Progress summary. - if e.PartNumber == 0 { - //update this atomic counter which is monitored by another go routine - //reporting numbers to the user - cca.setFirstPartOrdered() - } - e.CopyJobRequest.Transfers = []common.CopyTransfer{} - e.PartNumber++ - } - e.CopyJobRequest.Transfers = append(e.CopyJobRequest.Transfers, transfer) - return nil -} - -// we need to send a last part with isFinalPart set to true, along with whatever transfers that still haven't been sent -func (e *syncUploadEnumerator) dispatchFinalPart(cca *cookedSyncCmdArgs) error { - numberOfCopyTransfers := uint64(len(e.CopyJobRequest.Transfers)) - numberOfDeleteTransfers := uint64(len(e.DeleteJobRequest.Transfers)) - if numberOfCopyTransfers == 0 && numberOfDeleteTransfers == 0 { - glcm.Exit("cannot start job because there are no files to upload or delete. "+ - "The source and destination are in sync", 0) - return nil - } - // sendDeleteTransfers is an internal function which creates JobPartRequest for all the delete transfers enumerated. - // It creates requests for group of 10000 transfers. - sendDeleteTransfers := func() error { - currentCount := uint64(0) - // If the user agrees to delete the transfers, then break the entire deleteJobRequest into parts of 10000 size and send them - for numberOfDeleteTransfers > 0 { - // number of transfers in the current request can be either 10,000 or less than that. - numberOfTransfers := common.Iffuint64(numberOfDeleteTransfers > NumOfFilesPerDispatchJobPart, NumOfFilesPerDispatchJobPart, numberOfDeleteTransfers) - // create a copy of DeleteJobRequest - deleteJobRequest := e.DeleteJobRequest - // Reset the transfer list in the copy of DeleteJobRequest - deleteJobRequest.Transfers = []common.CopyTransfer{} - // Copy the transfers from currentCount till number of transfer calculated for current iteration - deleteJobRequest.Transfers = e.DeleteJobRequest.Transfers[currentCount : currentCount+numberOfTransfers] - // Set the part number - deleteJobRequest.PartNum = e.PartNumber - // Increment the part number - e.PartNumber++ - // Increment the current count - currentCount += numberOfTransfers - // Decrease the numberOfDeleteTransfer by the number Of transfers calculated for the current iteration - numberOfDeleteTransfers -= numberOfTransfers - // If the number of delete transfer is 0, it means it is the last part that needs to be sent. - // Set the IsFinalPart for the current request to true - if numberOfDeleteTransfers == 0 { - deleteJobRequest.IsFinalPart = true - } - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&deleteJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // If the part sent above was the first, then set setFirstPartOrdered, so that progress can be fetched. - if deleteJobRequest.PartNum == 0 { - cca.setFirstPartOrdered() - } - } - return nil - } - if numberOfCopyTransfers > 0 && numberOfDeleteTransfers > 0 { - var resp common.CopyJobPartOrderResponse - e.CopyJobRequest.PartNum = e.PartNumber - answer := "" - if cca.force { - answer = "y" - } else { - answer = glcm.Prompt(fmt.Sprintf("Sync has enumerated %v files to delete from destination. Do you want to delete these files ? Please confirm with y/n: ", numberOfDeleteTransfers)) - } - // read a line from stdin, if the answer is not yes, then is No, then ignore the transfers queued for deletion and continue - if !strings.EqualFold(answer, "y") { - e.CopyJobRequest.IsFinalPart = true - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - return nil - } - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // If the part sent above was the first, then setFirstPartOrdered, so that progress can be fetched. - if e.PartNumber == 0 { - cca.setFirstPartOrdered() - } - e.PartNumber++ - err := sendDeleteTransfers() - cca.isEnumerationComplete = true - return err - } else if numberOfCopyTransfers > 0 { - e.CopyJobRequest.IsFinalPart = true - e.CopyJobRequest.PartNum = e.PartNumber - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - cca.isEnumerationComplete = true - return nil - } - answer := "" - // If the user set the force flag to true, then prompt is not required and file will be deleted. - if cca.force { - answer = "y" - } else { - answer = glcm.Prompt(fmt.Sprintf("Sync has enumerated %v files to delete from destination. Do you want to delete these files ? Please confirm with y/n: ", numberOfDeleteTransfers)) - } - // read a line from stdin, if the answer is not yes, then is No, then ignore the transfers queued for deletion and continue - if !strings.EqualFold(answer, "y") { - return fmt.Errorf("cannot start job because there are no transfer to upload or delete. " + - "The source and destination are in sync") - } - error := sendDeleteTransfers() - cca.isEnumerationComplete = true - return error -} - -func (e *syncUploadEnumerator) listTheSourceIfRequired(cca *cookedSyncCmdArgs, p pipeline.Pipeline) (bool, error) { - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - util := copyHandlerUtil{} - - // attempt to parse the destination url - destinationUrl, err := url.Parse(cca.destination) - // the destination should have already been validated, it would be surprising if it cannot be parsed at this point - common.PanicIfErr(err) - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - destinationUrl = util.appendQueryParamToUrl(destinationUrl, cca.destinationSAS) - - blobUrl := azblob.NewBlobURL(*destinationUrl, p) - - // Get the files and directories for the given source pattern - listOfFilesAndDir, lofaderr := filepath.Glob(cca.source) - if lofaderr != nil { - return false, fmt.Errorf("error getting the files and directories for source pattern %s", cca.source) - } - - // Get the blob Properties - bProperties, bPropertiesError := blobUrl.GetProperties(ctx, azblob.BlobAccessConditions{}) - - // isSourceASingleFile is used to determine whether given source pattern represents single file or not - // If the source is a single file, this pointer will not be nil - // if it is nil, it means the source is a directory or list of file - var isSourceASingleFile os.FileInfo = nil - - if len(listOfFilesAndDir) == 0 { - fInfo, fError := os.Stat(listOfFilesAndDir[0]) - if fError != nil { - return false, fmt.Errorf("cannot get the information of the %s. Failed with error %s", listOfFilesAndDir[0], fError) - } - if fInfo.Mode().IsRegular() { - isSourceASingleFile = fInfo - } - } - - // sync only happens between the source and destination of same type i.e between blob and blob or between Directory and Virtual Folder / Container - // If the source is a file and destination is not a blob, sync fails. - if isSourceASingleFile != nil && bPropertiesError != nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between file %s and non blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - // If the source is a directory and destination is a blob - if isSourceASingleFile == nil && bPropertiesError == nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between directory %s and blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - - // If both source is a file and destination is a blob, then we need to do the comparison and queue the transfer if required. - if isSourceASingleFile != nil && bPropertiesError == nil { - blobName := destinationUrl.Path[strings.LastIndex(destinationUrl.Path, "/")+1:] - // Compare the blob name and file name - // blobName and filename should be same for sync to happen - if strings.Compare(blobName, isSourceASingleFile.Name()) != 0 { - glcm.Exit(fmt.Sprintf("sync cannot be done since blob %s and filename %s doesn't match", blobName, isSourceASingleFile.Name()), 1) - } - - // If the modified time of file local is not later than that of blob - // sync does not needs to happen. - if !isSourceASingleFile.ModTime().After(bProperties.LastModified()) { - glcm.Exit(fmt.Sprintf("blob %s and file %s already in sync", blobName, isSourceASingleFile.Name()), 1) - } - - e.addTransferToUpload(common.CopyTransfer{ - Source: cca.source, - Destination: util.stripSASFromBlobUrl(*destinationUrl).String(), - SourceSize: isSourceASingleFile.Size(), - LastModifiedTime: isSourceASingleFile.ModTime(), - }, cca) - return true, nil - } - - if len(listOfFilesAndDir) == 1 && !cca.recursive { - glcm.Exit(fmt.Sprintf("error performing between source %s and destination %s is a directory. recursive flag is turned off.", cca.source, cca.destination), 1) - } - // Get the source path without the wildcards - // This is defined since the files mentioned with exclude flag - // & include flag are relative to the Source - // If the source has wildcards, then files are relative to the - // parent source path which is the path of last directory in the source - // without wildcards - // For Example: src = "/home/user/dir1" parentSourcePath = "/home/user/dir1" - // For Example: src = "/home/user/dir*" parentSourcePath = "/home/user" - // For Example: src = "/home/*" parentSourcePath = "/home" - parentSourcePath := cca.source - wcIndex := util.firstIndexOfWildCard(parentSourcePath) - if wcIndex != -1 { - parentSourcePath = parentSourcePath[:wcIndex] - pathSepIndex := strings.LastIndex(parentSourcePath, common.AZCOPY_PATH_SEPARATOR_STRING) - parentSourcePath = parentSourcePath[:pathSepIndex] - } - - // Iterate through each file / dir inside the source - // and then checkAndQueue - for _, fileOrDir := range listOfFilesAndDir { - f, err := os.Stat(fileOrDir) - if err != nil { - glcm.Info(fmt.Sprintf("cannot get the file Info for %s. failed with error %s", fileOrDir, err.Error())) - } - // directories are uploaded only if recursive is on - if f.IsDir() && cca.recursive { - // walk goes through the entire directory tree - err = filepath.Walk(fileOrDir, func(pathToFile string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fileInfo.IsDir() { - return nil - } else { - // replace the OS path separator in pathToFile string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - pathToFile = strings.Replace(pathToFile, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - if util.resourceShouldBeExcluded(parentSourcePath, e.Exclude, pathToFile) { - e.SourceFilesToExclude[pathToFile] = f.ModTime() - return nil - } - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[pathToFile] = fileInfo.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) - } - return nil - }) - } else if !f.IsDir() { - // replace the OS path separator in fileOrDir string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - fileOrDir = strings.Replace(fileOrDir, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - if util.resourceShouldBeExcluded(parentSourcePath, e.Exclude, fileOrDir) { - e.SourceFilesToExclude[fileOrDir] = f.ModTime() - continue - } - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[fileOrDir] = f.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) - } - } - return false, nil -} - -// listDestinationAndCompare lists the blob under the destination mentioned and verifies whether the blob -// exists locally or not by checking the expected localPath of blob in the sourceFiles map. If the blob -// does exists, it compares the last modified time. If it does not exists, it queues the blob for deletion. -func (e *syncUploadEnumerator) listDestinationAndCompare(cca *cookedSyncCmdArgs, p pipeline.Pipeline) error { - util := copyHandlerUtil{} - - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - // rootPath is the path of source without wildCards - // sourcePattern is the filePath pattern inside the source - // For Example: cca.source = C:\a1\a* , so rootPath = C:\a1 and filePattern is a* - // This is to avoid enumerator to compare any file inside the destination directory - // that doesn't match the pattern - // For Example: cca.source = C:\a1\a* des = https://? - // Only files that follow pattern a* will be compared - rootPath, sourcePattern := util.sourceRootPathWithoutWildCards(cca.source) - //replace the os path separator with path separator "/" which is path separator for blobs - //sourcePattern = strings.Replace(sourcePattern, string(os.PathSeparator), "/", -1) - destinationUrl, err := url.Parse(cca.destination) - if err != nil { - return fmt.Errorf("error parsing the destinatio url") - } - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - destinationUrl = util.appendQueryParamToUrl(destinationUrl, cca.destinationSAS) - - blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) - blobURLPartsExtension := blobURLPartsExtension{blobUrlParts} - - containerUrl := util.getContainerUrl(blobUrlParts) - searchPrefix, _, _ := blobURLPartsExtension.searchPrefixFromBlobURL() - - containerBlobUrl := azblob.NewContainerURL(containerUrl, p) - - // Get the destination path without the wildcards - // This is defined since the files mentioned with exclude flag - // & include flag are relative to the Destination - // If the Destination has wildcards, then files are relative to the - // parent Destination path which is the path of last directory in the Destination - // without wildcards - // For Example: dst = "/home/user/dir1" parentSourcePath = "/home/user/dir1" - // For Example: dst = "/home/user/dir*" parentSourcePath = "/home/user" - // For Example: dst = "/home/*" parentSourcePath = "/home" - parentDestinationPath := blobUrlParts.BlobName - wcIndex := util.firstIndexOfWildCard(parentDestinationPath) - if wcIndex != -1 { - parentDestinationPath = parentDestinationPath[:wcIndex] - pathSepIndex := strings.LastIndex(parentDestinationPath, common.AZCOPY_PATH_SEPARATOR_STRING) - parentDestinationPath = parentDestinationPath[:pathSepIndex] - } - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerBlobUrl.ListBlobsFlatSegment(ctx, marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Segment.BlobItems { - // realtivePathofBlobLocally is the local path relative to source at which blob should be downloaded - // Example: cca.source ="C:\User1\user-1" cca.destination = "https:///virtual-dir?" blob name = "virtual-dir/a.txt" - // realtivePathofBlobLocally = virtual-dir/a.txt - realtivePathofBlobLocally := util.relativePathToRoot(searchPrefix, blobInfo.Name, '/') - - // check if the listed blob segment matches the sourcePath pattern - // if it does not comparison is not required - if !util.matchBlobNameAgainstPattern(sourcePattern, realtivePathofBlobLocally, cca.recursive) { - continue - } - - // Increment the number of files scanned at the destination. - atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) - - // calculate the expected local path of the blob - blobLocalPath := util.generateLocalPath(rootPath, realtivePathofBlobLocally) - - // If the files is found in the list of files to be excluded, then it is not compared - _, found := e.SourceFilesToExclude[blobLocalPath] - if found { - continue - } - // Check if the blob exists in the map of source Files. If the file is - // found, compare the modified time of the file against the blob's last - // modified time. If the modified time of file is later than the blob's - // modified time, then queue transfer for upload. If not, then delete - // blobLocalPath from the map of sourceFiles. - localFileTime, found := e.SourceFiles[blobLocalPath] - if found { - if localFileTime.After(blobInfo.Properties.LastModified) { - e.addTransferToUpload(common.CopyTransfer{ - Source: blobLocalPath, - Destination: util.stripSASFromBlobUrl(util.generateBlobUrl(containerUrl, blobInfo.Name)).String(), - SourceSize: *blobInfo.Properties.ContentLength, - }, cca) - } - delete(e.SourceFiles, blobLocalPath) - } else { - // If the blob is not found in the map of source Files, queue it for - // delete - e.addTransferToDelete(common.CopyTransfer{ - Source: util.stripSASFromBlobUrl(util.generateBlobUrl(containerUrl, blobInfo.Name)).String(), - Destination: "", // no destination in case of Delete JobPartOrder - SourceSize: *blobInfo.Properties.ContentLength, - }, cca) - } - } - marker = listBlob.NextMarker - } - return nil -} - -// queueSourceFilesForUpload -func (e *syncUploadEnumerator) queueSourceFilesForUpload(cca *cookedSyncCmdArgs) { - util := copyHandlerUtil{} - // rootPath will be the parent source directory before the first wildcard - // For Example: cca.source = C:\a\b* rootPath = C:\a - // For Example: cca.source = C:\*\a* rootPath = c:\ - // In case of no wildCard, rootPath is equal to the source directory - // This rootPath is effective when wildCards are provided - // Using this rootPath, path of file on blob is calculated - rootPath, _ := util.sourceRootPathWithoutWildCards(cca.source) - - // attempt to parse the destination url - destinationUrl, err := url.Parse(cca.destination) - // the destination should have already been validated, it would be surprising if it cannot be parsed at this point - common.PanicIfErr(err) - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - destinationUrl = util.appendQueryParamToUrl(destinationUrl, cca.destinationSAS) - - blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) - - for file, _ := range e.SourceFiles { - // get the file Info - f, err := os.Stat(file) - if err != nil { - glcm.Info(fmt.Sprintf("Error %s getting the file info for file %s", err.Error(), file)) - continue - } - // localfileRelativePath is the path of file relative to root directory - // Example1: rootPath = C:\User\user1\dir-1 fileAbsolutePath = C:\User\user1\dir-1\a.txt localfileRelativePath = \a.txt - // Example2: rootPath = C:\User\user1\dir-1 fileAbsolutePath = C:\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - localfileRelativePath := strings.Replace(file, rootPath, "", 1) - // remove the path separator at the start of relative path - if len(localfileRelativePath) > 0 && localfileRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { - localfileRelativePath = localfileRelativePath[1:] - } - // Appending the fileRelativePath to the destinationUrl - // root = C:\User\user1\dir-1 cca.destination = https:///? - // fileAbsolutePath = C:\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - // filedestinationUrl = https:////dir-2/a.txt? - filedestinationUrl, _ := util.appendBlobNameToUrl(blobUrlParts, localfileRelativePath) - - err = e.addTransferToUpload(common.CopyTransfer{ - Source: file, - Destination: util.stripSASFromBlobUrl(filedestinationUrl).String(), - LastModifiedTime: f.ModTime(), - SourceSize: f.Size(), - }, cca) - if err != nil { - glcm.Info(fmt.Sprintf("Error %s uploading transfer source :%s and destination %s", err.Error(), file, util.stripSASFromBlobUrl(filedestinationUrl).String())) - } - - } -} - -// this function accepts the list of files/directories to transfer and processes them -func (e *syncUploadEnumerator) enumerate(cca *cookedSyncCmdArgs) error { - ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - // Create the new azblob pipeline - p, err := createBlobPipeline(ctx, e.CredentialInfo) - if err != nil { - return err - } - - // Copying the JobId of sync job to individual copyJobRequest - e.CopyJobRequest.JobID = e.JobID - // Copying the FromTo of sync job to individual copyJobRequest - e.CopyJobRequest.FromTo = e.FromTo - - // set the sas of user given Source - e.CopyJobRequest.SourceSAS = e.SourceSAS - - // set the sas of user given destination - e.CopyJobRequest.DestinationSAS = e.DestinationSAS - - // Copying the JobId of sync job to individual deleteJobRequest. - e.DeleteJobRequest.JobID = e.JobID - // FromTo of DeleteJobRequest will be BlobTrash. - e.DeleteJobRequest.FromTo = common.EFromTo.BlobTrash() - - // For delete the source is the destination in case of sync upload - // For Example: source = /home/user destination = https://container/vd-1? - // For deleting the blobs, Source in Delete Job Source will be the blob url - // and source sas is the destination sas which is url sas. - // set the destination sas as the source sas - e.DeleteJobRequest.SourceSAS = e.DestinationSAS - - // Set the Log Level - e.CopyJobRequest.LogLevel = e.LogLevel - e.DeleteJobRequest.LogLevel = e.LogLevel - - // Set the force flag to true - e.CopyJobRequest.ForceWrite = true - - // Copy the sync Command String to the CopyJobPartRequest and DeleteJobRequest - e.CopyJobRequest.CommandString = e.CommandString - e.DeleteJobRequest.CommandString = e.CommandString - - // Set credential info properly - e.CopyJobRequest.CredentialInfo = e.CredentialInfo - e.DeleteJobRequest.CredentialInfo = e.CredentialInfo - - e.SourceFiles = make(map[string]time.Time) - - e.SourceFilesToExclude = make(map[string]time.Time) - - cca.waitUntilJobCompletion(false) - - // list the source files and store in the map. - // While listing the source files, it applies the exclude filter - // and stores them into a separate map "sourceFilesToExclude" - isSourceAFile, err := e.listTheSourceIfRequired(cca, p) - if err != nil { - return err - } - // isSourceAFile defines whether source is a file or not. - // If source is a file and destination is a blob, then destination doesn't needs to be compared against local. - if !isSourceAFile { - err = e.listDestinationAndCompare(cca, p) - if err != nil { - return err - } - } - e.queueSourceFilesForUpload(cca) - - // No Job Part has been dispatched, then dispatch the JobPart. - if e.PartNumber == 0 || - len(e.CopyJobRequest.Transfers) > 0 || - len(e.DeleteJobRequest.Transfers) > 0 { - err = e.dispatchFinalPart(cca) - if err != nil { - return err - } - //cca.waitUntilJobCompletion(true) - cca.setFirstPartOrdered() - } - cca.setScanningComplete() - return nil -} diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go new file mode 100644 index 000000000..e0ea3a2e9 --- /dev/null +++ b/cmd/zc_enumerator.go @@ -0,0 +1,174 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + "strings" + "time" +) + +// -------------------------------------- Component Definitions -------------------------------------- \\ +// the following interfaces and structs allow the sync enumerator +// to be generic and has as little duplicated code as possible + +// represent a local or remote resource object (ex: local file, blob, etc.) +// we can add more properties if needed, as this is easily extensible +type storedObject struct { + name string + lastModifiedTime time.Time + size int64 + md5 []byte + + // partial path relative to its root directory + // example: rootDir=/var/a/b/c fullPath=/var/a/b/c/d/e/f.pdf => relativePath=d/e/f.pdf name=f.pdf + relativePath string +} + +func (storedObject *storedObject) isMoreRecentThan(storedObject2 storedObject) bool { + return storedObject.lastModifiedTime.After(storedObject2.lastModifiedTime) +} + +// a constructor is used so that in case the storedObject has to change, the callers would get a compilation error +func newStoredObject(name string, relativePath string, lmt time.Time, size int64, md5 []byte) storedObject { + return storedObject{ + name: name, + relativePath: relativePath, + lastModifiedTime: lmt, + size: size, + md5: md5, + } +} + +// capable of traversing a structured resource like container or local directory +// pass each storedObject to the given objectProcessor if it passes all the filters +type resourceTraverser interface { + traverse(processor objectProcessor, filters []objectFilter) error +} + +// given a storedObject, process it accordingly +type objectProcessor func(storedObject storedObject) error + +// given a storedObject, verify if it satisfies the defined conditions +// if yes, return true +type objectFilter interface { + doesPass(storedObject storedObject) bool +} + +// -------------------------------------- Generic Enumerators -------------------------------------- \\ +// the following enumerators must be instantiated with configurations +// they define the work flow in the most generic terms + +type syncEnumerator struct { + // these allow us to go through the source and destination + // there is flexibility in which side we scan first, it could be either the source or the destination + primaryTraverser resourceTraverser + secondaryTraverser resourceTraverser + + // the results from the primary traverser would be stored here + objectIndexer *objectIndexer + + // general filters apply to both the primary and secondary traverser + filters []objectFilter + + // the processor that apply only to the secondary traverser + // it processes objects as scanning happens + // based on the data from the primary traverser stored in the objectIndexer + objectComparator objectProcessor + + // a finalizer that is always called if the enumeration finishes properly + finalize func() error +} + +func newSyncEnumerator(primaryTraverser, secondaryTraverser resourceTraverser, indexer *objectIndexer, + filters []objectFilter, comparator objectProcessor, finalize func() error) *syncEnumerator { + return &syncEnumerator{ + primaryTraverser: primaryTraverser, + secondaryTraverser: secondaryTraverser, + objectIndexer: indexer, + filters: filters, + objectComparator: comparator, + finalize: finalize, + } +} + +func (e *syncEnumerator) enumerate() (err error) { + // enumerate the primary resource and build lookup map + err = e.primaryTraverser.traverse(e.objectIndexer.store, e.filters) + if err != nil { + return + } + + // enumerate the secondary resource and as the objects pass the filters + // they will be passed to the object comparator + // which can process given objects based on what's already indexed + // note: transferring can start while scanning is ongoing + err = e.secondaryTraverser.traverse(e.objectComparator, e.filters) + if err != nil { + return + } + + // execute the finalize func which may perform useful clean up steps + err = e.finalize() + if err != nil { + return + } + + return +} + +// -------------------------------------- Helper Funcs -------------------------------------- \\ + +func passedFilters(filters []objectFilter, storedObject storedObject) bool { + if filters != nil && len(filters) > 0 { + // loop through the filters, if any of them fail, then return false + for _, filter := range filters { + if !filter.doesPass(storedObject) { + return false + } + } + } + + return true +} + +func processIfPassedFilters(filters []objectFilter, storedObject storedObject, processor objectProcessor) (err error) { + if passedFilters(filters, storedObject) { + err = processor(storedObject) + } + + return +} + +// storedObject names are useful for filters +func getObjectNameOnly(fullPath string) (nameOnly string) { + lastPathSeparator := strings.LastIndex(fullPath, common.AZCOPY_PATH_SEPARATOR_STRING) + + // if there is a path separator and it is not the last character + if lastPathSeparator > 0 && lastPathSeparator != len(fullPath)-1 { + // then we separate out the name of the storedObject + nameOnly = fullPath[lastPathSeparator+1:] + } else { + nameOnly = fullPath + } + + return +} diff --git a/cmd/zc_filter.go b/cmd/zc_filter.go new file mode 100644 index 000000000..9418836f6 --- /dev/null +++ b/cmd/zc_filter.go @@ -0,0 +1,100 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import "path" + +type excludeFilter struct { + pattern string +} + +func (f *excludeFilter) doesPass(storedObject storedObject) bool { + matched, err := path.Match(f.pattern, storedObject.name) + + // if the pattern failed to match with an error, then we assume the pattern is invalid + // and let it pass + if err != nil { + return true + } + + if matched { + return false + } + + return true +} + +func buildExcludeFilters(patterns []string) []objectFilter { + filters := make([]objectFilter, 0) + for _, pattern := range patterns { + if pattern != "" { + filters = append(filters, &excludeFilter{pattern: pattern}) + } + } + + return filters +} + +// design explanation: +// include filters are different from the exclude ones, which work together in the "AND" manner +// meaning and if an storedObject is rejected by any of the exclude filters, then it is rejected by all of them +// as a result, the exclude filters can be in their own struct, and work correctly +// on the other hand, include filters work in the "OR" manner +// meaning that if an storedObject is accepted by any of the include filters, then it is accepted by all of them +// consequently, all the include patterns must be stored together +type includeFilter struct { + patterns []string +} + +func (f *includeFilter) doesPass(storedObject storedObject) bool { + if len(f.patterns) == 0 { + return true + } + + for _, pattern := range f.patterns { + matched, err := path.Match(pattern, storedObject.name) + + // if the pattern failed to match with an error, then we assume the pattern is invalid + // and ignore it + if err != nil { + continue + } + + // if an storedObject is accepted by any of the include filters + // it is accepted + if matched { + return true + } + } + + return false +} + +func buildIncludeFilters(patterns []string) []objectFilter { + validPatterns := make([]string, 0) + for _, pattern := range patterns { + if pattern != "" { + validPatterns = append(validPatterns, pattern) + } + } + + return []objectFilter{&includeFilter{patterns: validPatterns}} +} diff --git a/cmd/zc_processor.go b/cmd/zc_processor.go new file mode 100644 index 000000000..d6ea6f498 --- /dev/null +++ b/cmd/zc_processor.go @@ -0,0 +1,128 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "fmt" + "github.com/Azure/azure-storage-azcopy/common" + "net/url" +) + +type copyTransferProcessor struct { + numOfTransfersPerPart int + copyJobTemplate *common.CopyJobPartOrderRequest + source string + destination string + + // specify whether source/destination object names need to be URL encoded before dispatching + shouldEscapeSourceObjectName bool + shouldEscapeDestinationObjectName bool + + // handles for progress tracking + reportFirstPartDispatched func() + reportFinalPartDispatched func() +} + +func newCopyTransferProcessor(copyJobTemplate *common.CopyJobPartOrderRequest, numOfTransfersPerPart int, + source string, destination string, shouldEscapeSourceObjectName bool, shouldEscapeDestinationObjectName bool, + reportFirstPartDispatched func(), reportFinalPartDispatched func()) *copyTransferProcessor { + return ©TransferProcessor{ + numOfTransfersPerPart: numOfTransfersPerPart, + copyJobTemplate: copyJobTemplate, + source: source, + destination: destination, + shouldEscapeSourceObjectName: shouldEscapeSourceObjectName, + shouldEscapeDestinationObjectName: shouldEscapeDestinationObjectName, + reportFirstPartDispatched: reportFirstPartDispatched, + reportFinalPartDispatched: reportFinalPartDispatched, + } +} + +func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject storedObject) (err error) { + if len(s.copyJobTemplate.Transfers) == s.numOfTransfersPerPart { + err = s.sendPartToSte() + if err != nil { + return err + } + + // reset the transfers buffer + s.copyJobTemplate.Transfers = []common.CopyTransfer{} + s.copyJobTemplate.PartNum++ + } + + // only append the transfer after we've checked and dispatched a part + // so that there is at least one transfer for the final part + s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ + Source: s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeSourceObjectName), + Destination: s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeDestinationObjectName), + SourceSize: storedObject.size, + LastModifiedTime: storedObject.lastModifiedTime, + ContentMD5: storedObject.md5, + }) + return nil +} + +func (s *copyTransferProcessor) escapeIfNecessary(path string, shouldEscape bool) string { + if shouldEscape { + return url.PathEscape(path) + } + + return path +} + +func (s *copyTransferProcessor) dispatchFinalPart() (copyJobInitiated bool, err error) { + numberOfCopyTransfers := len(s.copyJobTemplate.Transfers) + + // if the number of transfer to copy is 0 + // and no part was dispatched, then it means there is no work to do + if s.copyJobTemplate.PartNum == 0 && numberOfCopyTransfers == 0 { + return false, nil + } + + if numberOfCopyTransfers > 0 { + s.copyJobTemplate.IsFinalPart = true + err = s.sendPartToSte() + if err != nil { + return false, err + } + } + + if s.reportFinalPartDispatched != nil { + s.reportFinalPartDispatched() + } + return true, nil +} + +func (s *copyTransferProcessor) sendPartToSte() error { + var resp common.CopyJobPartOrderResponse + Rpc(common.ERpcCmd.CopyJobPartOrder(), s.copyJobTemplate, &resp) + if !resp.JobStarted { + return fmt.Errorf("copy job part order with JobId %s and part number %d failed to dispatch because %s", + s.copyJobTemplate.JobID, s.copyJobTemplate.PartNum, resp.ErrorMsg) + } + + // if the current part order sent to ste is 0, then alert the progress reporting routine + if s.copyJobTemplate.PartNum == 0 && s.reportFirstPartDispatched != nil { + s.reportFirstPartDispatched() + } + + return nil +} diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go new file mode 100644 index 000000000..b4e60f8d3 --- /dev/null +++ b/cmd/zc_traverser_blob.go @@ -0,0 +1,134 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "fmt" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + "net/url" + "strings" +) + +// allow us to iterate through a path pointing to the blob endpoint +type blobTraverser struct { + rawURL *url.URL + p pipeline.Pipeline + ctx context.Context + recursive bool + + // a generic function to notify that a new stored object has been enumerated + incrementEnumerationCounter func() +} + +func (t *blobTraverser) getPropertiesIfSingleBlob() (*azblob.BlobGetPropertiesResponse, bool) { + blobURL := azblob.NewBlobURL(*t.rawURL, t.p) + blobProps, blobPropertiesErr := blobURL.GetProperties(t.ctx, azblob.BlobAccessConditions{}) + + // if there was no problem getting the properties, it means that we are looking at a single blob + if blobPropertiesErr == nil && !gCopyUtil.doesBlobRepresentAFolder(blobProps.NewMetadata()) { + return blobProps, true + } + + return nil, false +} + +func (t *blobTraverser) traverse(processor objectProcessor, filters []objectFilter) (err error) { + blobUrlParts := azblob.NewBlobURLParts(*t.rawURL) + util := copyHandlerUtil{} + + // check if the url points to a single blob + blobProperties, isBlob := t.getPropertiesIfSingleBlob() + if isBlob { + storedObject := newStoredObject( + getObjectNameOnly(blobUrlParts.BlobName), + "", // relative path makes no sense when the full path already points to the file + blobProperties.LastModified(), + blobProperties.ContentLength(), + blobProperties.ContentMD5(), + ) + t.incrementEnumerationCounter() + return processIfPassedFilters(filters, storedObject, processor) + } + + // get the container URL so that we can list the blobs + containerRawURL := copyHandlerUtil{}.getContainerUrl(blobUrlParts) + containerURL := azblob.NewContainerURL(containerRawURL, t.p) + + // get the search prefix to aid in the listing + // example: for a url like https://test.blob.core.windows.net/test/foo/bar/bla + // the search prefix would be foo/bar/bla + searchPrefix := blobUrlParts.BlobName + + // append a slash if it is not already present + // example: foo/bar/bla becomes foo/bar/bla/ so that we only list children of the virtual directory + if searchPrefix != "" && !strings.HasSuffix(searchPrefix, common.AZCOPY_PATH_SEPARATOR_STRING) { + searchPrefix += common.AZCOPY_PATH_SEPARATOR_STRING + } + + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix + // TODO optimize for the case where recursive is off + listBlob, err := containerURL.ListBlobsFlatSegment(t.ctx, marker, + azblob.ListBlobsSegmentOptions{Prefix: searchPrefix, Details: azblob.BlobListingDetails{Metadata: true}}) + if err != nil { + return fmt.Errorf("cannot list blobs. Failed with error %s", err.Error()) + } + + // process the blobs returned in this result segment + for _, blobInfo := range listBlob.Segment.BlobItems { + // if the blob represents a hdi folder, then skip it + if util.doesBlobRepresentAFolder(blobInfo.Metadata) { + continue + } + + relativePath := strings.Replace(blobInfo.Name, searchPrefix, "", 1) + + // if recursive + if !t.recursive && strings.Contains(relativePath, common.AZCOPY_PATH_SEPARATOR_STRING) { + continue + } + + storedObject := storedObject{ + name: getObjectNameOnly(blobInfo.Name), + relativePath: relativePath, + lastModifiedTime: blobInfo.Properties.LastModified, + size: *blobInfo.Properties.ContentLength, + } + t.incrementEnumerationCounter() + processErr := processIfPassedFilters(filters, storedObject, processor) + if processErr != nil { + return processErr + } + } + + marker = listBlob.NextMarker + } + + return +} + +func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive bool, incrementEnumerationCounter func()) (t *blobTraverser) { + t = &blobTraverser{rawURL: rawURL, p: p, ctx: ctx, recursive: recursive, incrementEnumerationCounter: incrementEnumerationCounter} + return +} diff --git a/cmd/zc_traverser_local.go b/cmd/zc_traverser_local.go new file mode 100644 index 000000000..5be2714c7 --- /dev/null +++ b/cmd/zc_traverser_local.go @@ -0,0 +1,128 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "fmt" + "github.com/Azure/azure-storage-azcopy/common" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type localTraverser struct { + fullPath string + recursive bool + + // a generic function to notify that a new stored object has been enumerated + incrementEnumerationCounter func() +} + +func (t *localTraverser) traverse(processor objectProcessor, filters []objectFilter) (err error) { + singleFileInfo, isSingleFile, err := t.getInfoIfSingleFile() + + if err != nil { + return fmt.Errorf("cannot scan the path %s, please verify that it is a valid", t.fullPath) + } + + // if the path is a single file, then pass it through the filters and send to processor + if isSingleFile { + t.incrementEnumerationCounter() + err = processIfPassedFilters(filters, newStoredObject(singleFileInfo.Name(), + "", // relative path makes no sense when the full path already points to the file + singleFileInfo.ModTime(), singleFileInfo.Size(), nil), processor) + return + + } else { + if t.recursive { + err = filepath.Walk(t.fullPath, func(filePath string, fileInfo os.FileInfo, fileError error) error { + if fileError != nil { + return fileError + } + + // skip the subdirectories + if fileInfo.IsDir() { + return nil + } + + t.incrementEnumerationCounter() + return processIfPassedFilters(filters, newStoredObject(fileInfo.Name(), + strings.Replace(replacePathSeparators(filePath), t.fullPath+common.AZCOPY_PATH_SEPARATOR_STRING, + "", 1), fileInfo.ModTime(), fileInfo.Size(), nil), processor) + }) + + return + } else { + // if recursive is off, we only need to scan the files immediately under the fullPath + files, err := ioutil.ReadDir(t.fullPath) + if err != nil { + return err + } + + // go through the files and return if any of them fail to process + for _, singleFile := range files { + if singleFile.IsDir() { + continue + } + + t.incrementEnumerationCounter() + err = processIfPassedFilters(filters, newStoredObject(singleFile.Name(), singleFile.Name(), singleFile.ModTime(), singleFile.Size(), nil), processor) + + if err != nil { + return err + } + } + } + } + + return +} + +func replacePathSeparators(path string) string { + if os.PathSeparator != common.AZCOPY_PATH_SEPARATOR_CHAR { + return strings.Replace(path, string(os.PathSeparator), common.AZCOPY_PATH_SEPARATOR_STRING, -1) + } else { + return path + } +} + +func (t *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) { + fileInfo, err := os.Stat(t.fullPath) + + if err != nil { + return nil, false, err + } + + if fileInfo.IsDir() { + return nil, false, nil + } + + return fileInfo, true, nil +} + +func newLocalTraverser(fullPath string, recursive bool, incrementEnumerationCounter func()) *localTraverser { + traverser := localTraverser{ + fullPath: replacePathSeparators(fullPath), + recursive: recursive, + incrementEnumerationCounter: incrementEnumerationCounter} + return &traverser +} diff --git a/cmd/zt_generic_filter_test.go b/cmd/zt_generic_filter_test.go new file mode 100644 index 000000000..5ff23f623 --- /dev/null +++ b/cmd/zt_generic_filter_test.go @@ -0,0 +1,75 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + chk "gopkg.in/check.v1" +) + +type genericFilterSuite struct{} + +var _ = chk.Suite(&genericFilterSuite{}) + +func (s *genericFilterSuite) TestIncludeFilter(c *chk.C) { + // set up the filters + raw := rawSyncCmdArgs{} + includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") + includeFilter := buildIncludeFilters(includePatternList)[0] + + // test the positive cases + filesToPass := []string{"bla.pdf", "fancy.jpeg", "socool.jpeg.pdf", "exactName"} + for _, file := range filesToPass { + passed := includeFilter.doesPass(storedObject{name: file}) + c.Assert(passed, chk.Equals, true) + } + + // test the negative cases + filesNotToPass := []string{"bla.pdff", "fancyjpeg", "socool.jpeg.pdf.wut", "eexactName"} + for _, file := range filesNotToPass { + passed := includeFilter.doesPass(storedObject{name: file}) + c.Assert(passed, chk.Equals, false) + } +} + +func (s *genericFilterSuite) TestExcludeFilter(c *chk.C) { + // set up the filters + raw := rawSyncCmdArgs{} + excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") + excludeFilterList := buildExcludeFilters(excludePatternList) + + // test the positive cases + filesToPass := []string{"bla.pdfe", "fancy.jjpeg", "socool.png", "eexactName"} + for _, file := range filesToPass { + dummyProcessor := &dummyProcessor{} + err := processIfPassedFilters(excludeFilterList, storedObject{name: file}, dummyProcessor.process) + c.Assert(err, chk.IsNil) + c.Assert(len(dummyProcessor.record), chk.Equals, 1) + } + + // test the negative cases + filesToNotPass := []string{"bla.pdf", "fancy.jpeg", "socool.jpeg.pdf", "exactName"} + for _, file := range filesToNotPass { + dummyProcessor := &dummyProcessor{} + err := processIfPassedFilters(excludeFilterList, storedObject{name: file}, dummyProcessor.process) + c.Assert(err, chk.IsNil) + c.Assert(len(dummyProcessor.record), chk.Equals, 0) + } +} diff --git a/cmd/zt_generic_processor_test.go b/cmd/zt_generic_processor_test.go new file mode 100644 index 000000000..611053d64 --- /dev/null +++ b/cmd/zt_generic_processor_test.go @@ -0,0 +1,148 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" + "path/filepath" + "time" +) + +type genericProcessorSuite struct{} + +var _ = chk.Suite(&genericProcessorSuite{}) + +type processorTestSuiteHelper struct{} + +// return a list of sample entities +func (processorTestSuiteHelper) getSampleObjectList() []storedObject { + return []storedObject{ + {name: "file1", relativePath: "file1", lastModifiedTime: time.Now()}, + {name: "file2", relativePath: "file2", lastModifiedTime: time.Now()}, + {name: "file3", relativePath: "sub1/file3", lastModifiedTime: time.Now()}, + {name: "file4", relativePath: "sub1/file4", lastModifiedTime: time.Now()}, + {name: "file5", relativePath: "sub1/sub2/file5", lastModifiedTime: time.Now()}, + {name: "file6", relativePath: "sub1/sub2/file6", lastModifiedTime: time.Now()}, + } +} + +// given a list of entities, return the relative paths in a list, to help with validations +func (processorTestSuiteHelper) getExpectedTransferFromStoredObjectList(storedObjectList []storedObject) []string { + expectedTransfers := make([]string, 0) + for _, storedObject := range storedObjectList { + expectedTransfers = append(expectedTransfers, storedObject.relativePath) + } + + return expectedTransfers +} + +func (processorTestSuiteHelper) getCopyJobTemplate() *common.CopyJobPartOrderRequest { + return &common.CopyJobPartOrderRequest{} +} + +func (s *genericProcessorSuite) TestCopyTransferProcessorMultipleFiles(c *chk.C) { + bsu := getBSU() + + // set up source and destination + containerURL, _ := getContainerURL(c, bsu) + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // exercise the processor + sampleObjects := processorTestSuiteHelper{}.getSampleObjectList() + for _, numOfParts := range []int{1, 3} { + numOfTransfersPerPart := len(sampleObjects) / numOfParts + copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), numOfTransfersPerPart, + containerURL.String(), dstDirName, false, false, nil, nil) + + // go through the objects and make sure they are processed without error + for _, storedObject := range sampleObjects { + err := copyProcessor.scheduleCopyTransfer(storedObject) + c.Assert(err, chk.IsNil) + } + + // make sure everything has been dispatched apart from the final one + c.Assert(copyProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(numOfParts-1)) + + // dispatch final part + jobInitiated, err := copyProcessor.dispatchFinalPart() + c.Assert(jobInitiated, chk.Equals, true) + c.Assert(err, chk.IsNil) + + // assert the right transfers were scheduled + validateTransfersAreScheduled(c, containerURL.String(), false, dstDirName, false, + processorTestSuiteHelper{}.getExpectedTransferFromStoredObjectList(sampleObjects), mockedRPC) + + mockedRPC.reset() + } +} + +func (s *genericProcessorSuite) TestCopyTransferProcessorSingleFile(c *chk.C) { + bsu := getBSU() + containerURL, _ := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with a single blob + blobList := []string{"singlefile101"} + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + c.Assert(containerURL, chk.NotNil) + + // set up the directory with a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobList[0] + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // set up the processor + blobURL := containerURL.NewBlockBlobURL(blobList[0]).String() + copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), 2, + blobURL, filepath.Join(dstDirName, dstFileName), false, false, nil, nil) + + // exercise the copy transfer processor + storedObject := newStoredObject(blobList[0], "", time.Now(), 0, nil) + err := copyProcessor.scheduleCopyTransfer(storedObject) + c.Assert(err, chk.IsNil) + + // no part should have been dispatched + c.Assert(copyProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(0)) + + // dispatch final part + jobInitiated, err := copyProcessor.dispatchFinalPart() + c.Assert(jobInitiated, chk.Equals, true) + + // In cases of syncing file to file, the source and destination are empty because this info is already in the root + // path. + c.Assert(mockedRPC.transfers[0].Source, chk.Equals, "") + c.Assert(mockedRPC.transfers[0].Destination, chk.Equals, "") + + // assert the right transfers were scheduled + validateTransfersAreScheduled(c, blobURL, false, filepath.Join(dstDirName, dstFileName), false, + []string{""}, mockedRPC) +} diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go new file mode 100644 index 000000000..ba3052705 --- /dev/null +++ b/cmd/zt_generic_traverser_test.go @@ -0,0 +1,190 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-azcopy/ste" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "path/filepath" + "strings" + "time" +) + +type genericTraverserSuite struct{} + +var _ = chk.Suite(&genericTraverserSuite{}) + +// validate traversing a single blob and a single file +// compare that blob and local traversers get consistent results +func (s *genericTraverserSuite) TestTraverserWithSingleObject(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // test two scenarios, either blob is at the root virtual dir, or inside sub virtual dirs + for _, blobName := range []string{"sub1/sub2/singleblobisbest", "nosubsingleblob"} { + // set up the container with a single blob + blobList := []string{blobName} + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + c.Assert(containerURL, chk.NotNil) + + // set up the directory as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // construct a local traverser + localTraverser := newLocalTraverser(filepath.Join(dstDirName, dstFileName), false, func() {}) + + // invoke the local traversal with a dummy processor + localDummyProcessor := dummyProcessor{} + err := localTraverser.traverse(localDummyProcessor.process, nil) + c.Assert(err, chk.IsNil) + c.Assert(len(localDummyProcessor.record), chk.Equals, 1) + + // construct a blob traverser + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, false, func() {}) + + // invoke the blob traversal with a dummy processor + blobDummyProcessor := dummyProcessor{} + err = blobTraverser.traverse(blobDummyProcessor.process, nil) + c.Assert(err, chk.IsNil) + c.Assert(len(blobDummyProcessor.record), chk.Equals, 1) + + // assert the important info are correct + c.Assert(localDummyProcessor.record[0].name, chk.Equals, blobDummyProcessor.record[0].name) + c.Assert(localDummyProcessor.record[0].relativePath, chk.Equals, blobDummyProcessor.record[0].relativePath) + } +} + +// validate traversing a container and a local directory containing the same objects +// compare that blob and local traversers get consistent results +func (s *genericTraverserSuite) TestTraverserContainerAndLocalDirectory(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with numerous blobs + fileList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + c.Assert(containerURL, chk.NotNil) + + // set up the destination with a folder that have the exact same files + time.Sleep(2 * time.Second) // make the lmts of local files newer + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, fileList) + + // test two scenarios, either recursive or not + for _, isRecursiveOn := range []bool{true, false} { + // construct a local traverser + localTraverser := newLocalTraverser(dstDirName, isRecursiveOn, func() {}) + + // invoke the local traversal with an indexer + // so that the results are indexed for easy validation + localIndexer := newObjectIndexer() + err := localTraverser.traverse(localIndexer.store, nil) + c.Assert(err, chk.IsNil) + + // construct a blob traverser + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + blobTraverser := newBlobTraverser(&rawContainerURLWithSAS, p, ctx, isRecursiveOn, func() {}) + + // invoke the local traversal with a dummy processor + blobDummyProcessor := dummyProcessor{} + err = blobTraverser.traverse(blobDummyProcessor.process, nil) + c.Assert(err, chk.IsNil) + + // make sure the results are the same + c.Assert(len(blobDummyProcessor.record), chk.Equals, len(localIndexer.indexMap)) + for _, storedObject := range blobDummyProcessor.record { + correspondingLocalFile, present := localIndexer.indexMap[storedObject.relativePath] + + c.Assert(present, chk.Equals, true) + c.Assert(correspondingLocalFile.name, chk.Equals, storedObject.name) + c.Assert(correspondingLocalFile.isMoreRecentThan(storedObject), chk.Equals, true) + + if !isRecursiveOn { + c.Assert(strings.Contains(storedObject.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + } + } +} + +// validate traversing a virtual and a local directory containing the same objects +// compare that blob and local traversers get consistent results +func (s *genericTraverserSuite) TestTraverserWithVirtualAndLocalDirectory(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with numerous blobs + virDirName := "virdir" + fileList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, virDirName+"/") + c.Assert(containerURL, chk.NotNil) + + // set up the destination with a folder that have the exact same files + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, fileList) + + // test two scenarios, either recursive or not + for _, isRecursiveOn := range []bool{true, false} { + // construct a local traverser + localTraverser := newLocalTraverser(filepath.Join(dstDirName, virDirName), isRecursiveOn, func() {}) + + // invoke the local traversal with an indexer + // so that the results are indexed for easy validation + localIndexer := newObjectIndexer() + err := localTraverser.traverse(localIndexer.store, nil) + c.Assert(err, chk.IsNil) + + // construct a blob traverser + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) + blobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, func() {}) + + // invoke the local traversal with a dummy processor + blobDummyProcessor := dummyProcessor{} + err = blobTraverser.traverse(blobDummyProcessor.process, nil) + c.Assert(err, chk.IsNil) + + // make sure the results are the same + c.Assert(len(blobDummyProcessor.record), chk.Equals, len(localIndexer.indexMap)) + for _, storedObject := range blobDummyProcessor.record { + correspondingLocalFile, present := localIndexer.indexMap[storedObject.relativePath] + + c.Assert(present, chk.Equals, true) + c.Assert(correspondingLocalFile.name, chk.Equals, storedObject.name) + c.Assert(correspondingLocalFile.isMoreRecentThan(storedObject), chk.Equals, true) + + if !isRecursiveOn { + c.Assert(strings.Contains(storedObject.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + } + } +} diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go new file mode 100644 index 000000000..4c5cc6f45 --- /dev/null +++ b/cmd/zt_interceptors_for_test.go @@ -0,0 +1,102 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + "time" +) + +// the interceptor gathers/saves the job part orders for validation +type interceptor struct { + transfers []common.CopyTransfer + lastRequest interface{} +} + +func (i *interceptor) intercept(cmd common.RpcCmd, request interface{}, response interface{}) { + switch cmd { + case common.ERpcCmd.CopyJobPartOrder(): + // cache the transfers + copyRequest := *request.(*common.CopyJobPartOrderRequest) + i.transfers = append(i.transfers, copyRequest.Transfers...) + i.lastRequest = request + + // mock the result + *(response.(*common.CopyJobPartOrderResponse)) = common.CopyJobPartOrderResponse{JobStarted: true} + + case common.ERpcCmd.ListSyncJobSummary(): + copyRequest := *request.(*common.CopyJobPartOrderRequest) + + // fake the result saying that job is already completed + // doing so relies on the mockedLifecycleManager not quitting the application + *(response.(*common.ListSyncJobSummaryResponse)) = common.ListSyncJobSummaryResponse{ + Timestamp: time.Now().UTC(), + JobID: copyRequest.JobID, + ErrorMsg: "", + JobStatus: common.EJobStatus.Completed(), + CompleteJobOrdered: true, + FailedTransfers: []common.TransferDetail{}, + } + case common.ERpcCmd.ListJobs(): + case common.ERpcCmd.ListJobSummary(): + case common.ERpcCmd.ListJobTransfers(): + case common.ERpcCmd.PauseJob(): + case common.ERpcCmd.CancelJob(): + case common.ERpcCmd.ResumeJob(): + case common.ERpcCmd.GetJobFromTo(): + fallthrough + default: + panic("RPC mock not implemented") + } +} + +func (i *interceptor) init() { + // mock out the lifecycle manager so that it can no longer terminate the application + glcm = mockedLifecycleManager{} +} + +func (i *interceptor) reset() { + i.transfers = make([]common.CopyTransfer, 0) + i.lastRequest = nil +} + +// this lifecycle manager substitute does not perform any action +type mockedLifecycleManager struct{} + +func (mockedLifecycleManager) Progress(common.OutputBuilder) {} +func (mockedLifecycleManager) Init(common.OutputBuilder) {} +func (mockedLifecycleManager) Info(string) {} +func (mockedLifecycleManager) Prompt(string) string { return "" } +func (mockedLifecycleManager) Exit(common.OutputBuilder, common.ExitCode) {} +func (mockedLifecycleManager) Error(string) {} +func (mockedLifecycleManager) SurrenderControl() {} +func (mockedLifecycleManager) InitiateProgressReporting(common.WorkController, bool) {} +func (mockedLifecycleManager) GetEnvironmentVariable(common.EnvironmentVariable) string { return "" } +func (mockedLifecycleManager) SetOutputFormat(common.OutputFormat) {} + +type dummyProcessor struct { + record []storedObject +} + +func (d *dummyProcessor) process(storedObject storedObject) (err error) { + d.record = append(d.record, storedObject) + return +} diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go new file mode 100644 index 000000000..52a7d7fc7 --- /dev/null +++ b/cmd/zt_scenario_helpers_for_test.go @@ -0,0 +1,245 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +const defaultFileSize = 1024 + +type scenarioHelper struct{} + +var specialNames = []string{ + "打麻将.txt", + "wow such space so much space", + "saywut.pdf?yo=bla&WUWUWU=foo&sig=yyy", + "coração", + "আপনার নাম কি", + "%4509%4254$85140&", + "Donaudampfschifffahrtselektrizitätenhauptbetriebswerkbauunterbeamtengesellschaft", + "お名前は何ですか", + "Adın ne", + "як вас звати", +} + +func (scenarioHelper) generateLocalDirectory(c *chk.C) (dstDirName string) { + dstDirName, err := ioutil.TempDir("", "AzCopySyncDownload") + c.Assert(err, chk.IsNil) + return +} + +// create a test file +func (scenarioHelper) generateFile(filePath string, fileSize int) ([]byte, error) { + // generate random data + _, bigBuff := getRandomDataAndReader(fileSize) + + // create all parent directories + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + if err != nil { + return nil, err + } + + // write to file and return the data + err = ioutil.WriteFile(filePath, bigBuff, 0666) + return bigBuff, err +} + +func (s scenarioHelper) generateRandomLocalFiles(c *chk.C, dirPath string, prefix string) (fileList []string) { + fileList = make([]string, 50) + for i := 0; i < 10; i++ { + batch := []string{ + generateName(prefix + "top"), + generateName(prefix + "sub1/"), + generateName(prefix + "sub2/"), + generateName(prefix + "sub1/sub3/sub5/"), + generateName(prefix + specialNames[i]), + } + + for j, name := range batch { + fileList[5*i+j] = name + _, err := s.generateFile(filepath.Join(dirPath, name), defaultFileSize) + c.Assert(err, chk.IsNil) + } + } + + // sleep a bit so that the files' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) + return +} + +func (s scenarioHelper) generateFilesFromList(c *chk.C, dirPath string, fileList []string) { + for _, fileName := range fileList { + _, err := s.generateFile(filepath.Join(dirPath, fileName), defaultFileSize) + c.Assert(err, chk.IsNil) + } + + // sleep a bit so that the files' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) +} + +// make 50 blobs with random names +// 10 of them at the top level +// 10 of them in sub dir "sub1" +// 10 of them in sub dir "sub2" +// 10 of them in deeper sub dir "sub1/sub3/sub5" +// 10 of them with special characters +func (scenarioHelper) generateCommonRemoteScenario(c *chk.C, containerURL azblob.ContainerURL, prefix string) (blobList []string) { + blobList = make([]string, 50) + + for i := 0; i < 10; i++ { + _, blobName1 := createNewBlockBlob(c, containerURL, prefix+"top") + _, blobName2 := createNewBlockBlob(c, containerURL, prefix+"sub1/") + _, blobName3 := createNewBlockBlob(c, containerURL, prefix+"sub2/") + _, blobName4 := createNewBlockBlob(c, containerURL, prefix+"sub1/sub3/sub5/") + _, blobName5 := createNewBlockBlob(c, containerURL, prefix+specialNames[i]) + + blobList[5*i] = blobName1 + blobList[5*i+1] = blobName2 + blobList[5*i+2] = blobName3 + blobList[5*i+3] = blobName4 + blobList[5*i+4] = blobName5 + } + + // sleep a bit so that the blobs' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) + return +} + +// create the demanded blobs +func (scenarioHelper) generateBlobs(c *chk.C, containerURL azblob.ContainerURL, blobList []string) { + for _, blobName := range blobList { + blob := containerURL.NewBlockBlobURL(blobName) + cResp, err := blob.Upload(ctx, strings.NewReader(blockBlobDefaultData), azblob.BlobHTTPHeaders{}, + nil, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + c.Assert(cResp.StatusCode(), chk.Equals, 201) + } + + // sleep a bit so that the blobs' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) +} + +// Golang does not have sets, so we have to use a map to fulfill the same functionality +func (scenarioHelper) convertListToMap(list []string) map[string]int { + lookupMap := make(map[string]int) + for _, entryName := range list { + lookupMap[entryName] = 0 + } + + return lookupMap +} + +func (scenarioHelper) getRawContainerURLWithSAS(c *chk.C, containerName string) url.URL { + credential, err := getGenericCredential("") + c.Assert(err, chk.IsNil) + containerURLWithSAS := getContainerURLWithSAS(c, *credential, containerName) + return containerURLWithSAS.URL() +} + +func (scenarioHelper) getRawBlobURLWithSAS(c *chk.C, containerName string, blobName string) url.URL { + credential, err := getGenericCredential("") + c.Assert(err, chk.IsNil) + containerURLWithSAS := getContainerURLWithSAS(c, *credential, containerName) + blobURLWithSAS := containerURLWithSAS.NewBlockBlobURL(blobName) + return blobURLWithSAS.URL() +} + +func (scenarioHelper) blobExists(blobURL azblob.BlobURL) bool { + _, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + if err == nil { + return true + } + return false +} + +func runSyncAndVerify(c *chk.C, raw rawSyncCmdArgs, verifier func(err error)) { + // the simulated user input should parse properly + cooked, err := raw.cook() + c.Assert(err, chk.IsNil) + + // the enumeration ends when process() returns + err = cooked.process() + + // the err is passed to verified, which knows whether it is expected or not + verifier(err) +} + +func validateUploadTransfersAreScheduled(c *chk.C, srcDirName string, dstDirName string, expectedTransfers []string, mockedRPC interceptor) { + validateTransfersAreScheduled(c, srcDirName, false, dstDirName, true, expectedTransfers, mockedRPC) +} + +func validateDownloadTransfersAreScheduled(c *chk.C, srcDirName string, dstDirName string, expectedTransfers []string, mockedRPC interceptor) { + validateTransfersAreScheduled(c, srcDirName, true, dstDirName, false, expectedTransfers, mockedRPC) +} + +func validateTransfersAreScheduled(c *chk.C, srcDirName string, isSrcEncoded bool, dstDirName string, isDstEncoded bool, expectedTransfers []string, mockedRPC interceptor) { + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(expectedTransfers)) + + // validate that the right transfers were sent + lookupMap := scenarioHelper{}.convertListToMap(expectedTransfers) + for _, transfer := range mockedRPC.transfers { + srcRelativeFilePath := strings.Replace(transfer.Source, srcDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + dstRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + + if isSrcEncoded { + srcRelativeFilePath, _ = url.PathUnescape(srcRelativeFilePath) + } + + if isDstEncoded { + dstRelativeFilePath, _ = url.PathUnescape(dstRelativeFilePath) + } + + // the relative paths should be equal + c.Assert(srcRelativeFilePath, chk.Equals, dstRelativeFilePath) + + // look up the source from the expected transfers, make sure it exists + _, srcExist := lookupMap[dstRelativeFilePath] + c.Assert(srcExist, chk.Equals, true) + + // look up the destination from the expected transfers, make sure it exists + _, dstExist := lookupMap[dstRelativeFilePath] + c.Assert(dstExist, chk.Equals, true) + } +} + +func getDefaultRawInput(src, dst string) rawSyncCmdArgs { + deleteDestination := common.EDeleteDestination.True() + + return rawSyncCmdArgs{ + src: src, + dst: dst, + recursive: true, + logVerbosity: defaultLogVerbosityForSync, + deleteDestination: deleteDestination.String(), + md5ValidationOption: common.DefaultHashValidationOption.String(), + } +} diff --git a/cmd/zt_sync_comparator_test.go b/cmd/zt_sync_comparator_test.go new file mode 100644 index 000000000..3a8a869aa --- /dev/null +++ b/cmd/zt_sync_comparator_test.go @@ -0,0 +1,139 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + chk "gopkg.in/check.v1" + "time" +) + +type syncComparatorSuite struct{} + +var _ = chk.Suite(&syncComparatorSuite{}) + +func (s *syncComparatorSuite) TestSyncSourceComparator(c *chk.C) { + dummyCopyScheduler := dummyProcessor{} + srcMD5 := []byte{'s'} + destMD5 := []byte{'d'} + + // set up the indexer as well as the source comparator + indexer := newObjectIndexer() + sourceComparator := newSyncSourceComparator(indexer, dummyCopyScheduler.process) + + // create a sample destination object + sampleDestinationObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now(), md5: destMD5} + + // test the comparator in case a given source object is not present at the destination + // meaning no entry in the index, so the comparator should pass the given object to schedule a transfer + compareErr := sourceComparator.processIfNecessary(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now(), md5: srcMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // check the source object was indeed scheduled + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 1) + c.Assert(dummyCopyScheduler.record[0].md5, chk.DeepEquals, srcMD5) + + // reset the processor so that it's empty + dummyCopyScheduler = dummyProcessor{} + + // test the comparator in case a given source object is present at the destination + // and it has a later modified time, so the comparator should pass the give object to schedule a transfer + err := indexer.store(sampleDestinationObject) + c.Assert(err, chk.IsNil) + compareErr = sourceComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(time.Hour), md5: srcMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // check the source object was indeed scheduled + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 1) + c.Assert(dummyCopyScheduler.record[0].md5, chk.DeepEquals, srcMD5) + c.Assert(len(indexer.indexMap), chk.Equals, 0) + + // reset the processor so that it's empty + dummyCopyScheduler = dummyProcessor{} + + // test the comparator in case a given source object is present at the destination + // but is has an earlier modified time compared to the one at the destination + // meaning that the source object is considered stale, so no transfer should be scheduled + err = indexer.store(sampleDestinationObject) + c.Assert(err, chk.IsNil) + compareErr = sourceComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour), md5: srcMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // check no source object was scheduled + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 0) + c.Assert(len(indexer.indexMap), chk.Equals, 0) +} + +func (s *syncComparatorSuite) TestSyncDestinationComparator(c *chk.C) { + dummyCopyScheduler := dummyProcessor{} + dummyCleaner := dummyProcessor{} + srcMD5 := []byte{'s'} + destMD5 := []byte{'d'} + + // set up the indexer as well as the destination comparator + indexer := newObjectIndexer() + destinationComparator := newSyncDestinationComparator(indexer, dummyCopyScheduler.process, dummyCleaner.process) + + // create a sample source object + sampleSourceObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now(), md5: srcMD5} + + // test the comparator in case a given destination object is not present at the source + // meaning it is an extra file that needs to be deleted, so the comparator should pass the given object to the destinationCleaner + compareErr := destinationComparator.processIfNecessary(storedObject{name: "only_at_dst", relativePath: "only_at_dst", lastModifiedTime: time.Now(), md5: destMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // verify that destination object is being deleted + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 0) + c.Assert(len(dummyCleaner.record), chk.Equals, 1) + c.Assert(dummyCleaner.record[0].md5, chk.DeepEquals, destMD5) + + // reset dummy processors + dummyCopyScheduler = dummyProcessor{} + dummyCleaner = dummyProcessor{} + + // test the comparator in case a given destination object is present at the source + // and it has a later modified time, since the source data is stale, + // no transfer happens + err := indexer.store(sampleSourceObject) + c.Assert(err, chk.IsNil) + compareErr = destinationComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(time.Hour), md5: destMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // verify that the source object is scheduled for transfer + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 0) + c.Assert(len(dummyCleaner.record), chk.Equals, 0) + + // reset dummy processors + dummyCopyScheduler = dummyProcessor{} + dummyCleaner = dummyProcessor{} + + // test the comparator in case a given destination object is present at the source + // but is has an earlier modified time compared to the one at the source + // meaning that the source object should be transferred since the destination object is stale + err = indexer.store(sampleSourceObject) + c.Assert(err, chk.IsNil) + compareErr = destinationComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour), md5: destMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // verify that there's no transfer & no deletes + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 1) + c.Assert(dummyCopyScheduler.record[0].md5, chk.DeepEquals, srcMD5) + c.Assert(len(dummyCleaner.record), chk.Equals, 0) +} diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go new file mode 100644 index 000000000..929cef767 --- /dev/null +++ b/cmd/zt_sync_download_test.go @@ -0,0 +1,545 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "bytes" + "context" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "io/ioutil" + "path/filepath" + "strings" +) + +const ( + defaultLogVerbosityForSync = "WARNING" +) + +// regular blob->file sync +func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { + bsu := getBSU() + + for _, blobName := range []string{"singleblobisbest", "打麻将.txt", "%4509%4254$85140&"} { + // set up the container with a single blob + blobList := []string{blobName} + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // the file was created after the blob, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // recreate the blob to have a later last modified time + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + validateDownloadTransfersAreScheduled(c, containerURL.NewBlobURL(blobName).String(), + filepath.Join(dstDirName, dstFileName), []string{""}, mockedRPC) + }) + } +} + +// regular container->directory sync but destination is empty, so everything has to be transferred +func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(blobList)) + + // validate that the right transfers were sent + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) + + // turn off recursive, this time only top blobs should be transferred + raw.recursive = false + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + c.Assert(len(mockedRPC.transfers), chk.Not(chk.Equals), len(blobList)) + + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + c.Assert(strings.Contains(localRelativeFilePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + }) +} + +// regular container->directory sync but destination is identical to the source, transfers are scheduled based on lmt +func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination with a folder that have the exact same files + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // refresh the blobs' last modified time so that they are newer + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) +} + +// regular container->directory sync where destination is missing some files from source, and also has some extra files +func (s *cmdIntegrationSuite) TestSyncDownloadWithMismatchedDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination with a folder that have half of the files from source + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList[0:len(blobList)/2]) + scenarioHelper{}.generateFilesFromList(c, dstDirName, []string{"extraFile1.pdf, extraFile2.txt"}) + expectedOutput := blobList[len(blobList)/2:] // the missing half of source files should be transferred + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, expectedOutput, mockedRPC) + + // make sure the extra files were deleted + currentDstFileList, err := ioutil.ReadDir(dstDirName) + extraFilesFound := false + for _, file := range currentDstFileList { + if strings.Contains(file.Name(), "extra") { + extraFilesFound = true + } + } + + c.Assert(extraFilesFound, chk.Equals, false) + }) +} + +// include flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncDownloadWithIncludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // add special blobs that we wish to include + blobsToInclude := []string{"important.pdf", "includeSub/amazing.jpeg", "exactName"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.include = includeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) + }) +} + +// exclude flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncDownloadWithExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // add special blobs that we wish to exclude + blobsToExclude := []string{"notGood.pdf", "excludeSub/lame.jpeg", "exactName"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToExclude) + excludeString := "*.pdf;*.jpeg;exactName" + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) +} + +// include and exclude flag can work together to limit the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncDownloadWithIncludeAndExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // add special blobs that we wish to include + blobsToInclude := []string{"important.pdf", "includeSub/amazing.jpeg"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // add special blobs that we wish to exclude + // note that the excluded files also match the include string + blobsToExclude := []string{"sorry.pdf", "exclude/notGood.jpeg", "exactName", "sub/exactName"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToExclude) + excludeString := "so*;not*;exactName" + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.include = includeString + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) + }) +} + +// validate the bug fix for this scenario +func (s *cmdIntegrationSuite) TestSyncDownloadWithMissingDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination as a missing folder + dstDirName := filepath.Join(scenarioHelper{}.generateLocalDirectory(c), "imbatman") + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + // error should not be nil, but the app should not crash either + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// there is a type mismatch between the source and destination +func (s *cmdIntegrationSuite) TestSyncMismatchContainerAndFile(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobList[0] + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // reverse the source and destination + raw = getDefaultRawInput(filepath.Join(dstDirName, dstFileName), rawContainerURLWithSAS.String()) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// there is a type mismatch between the source and destination +func (s *cmdIntegrationSuite) TestSyncMismatchBlobAndDirectory(c *chk.C) { + bsu := getBSU() + + // set up the container with a single blob + blobName := "singleblobisbest" + blobList := []string{blobName} + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up the destination as a directory + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), dstDirName) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // reverse the source and destination + raw = getDefaultRawInput(dstDirName, rawBlobURLWithSAS.String()) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// download a blob representing an ADLS directory to a local file +// we should recognize that there is a type mismatch +func (s *cmdIntegrationSuite) TestSyncDownloadADLSDirectoryTypeMismatch(c *chk.C) { + bsu := getBSU() + blobName := "adlsdir" + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, []string{blobName}) + + // set up the container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // create a single blob that represents an ADLS directory + _, err := containerURL.NewBlockBlobURL(blobName).Upload(context.Background(), bytes.NewReader(nil), + azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobName) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // the file was created after the blob, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// adls directory -> local directory sync +// we should download every blob except the blob representing the directory +func (s *cmdIntegrationSuite) TestSyncDownloadWithADLSDirectory(c *chk.C) { + bsu := getBSU() + adlsDirName := "adlsdir" + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, adlsDirName+"/") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // create a single blob that represents the ADLS directory + dirBlob := containerURL.NewBlockBlobURL(adlsDirName) + _, err := dirBlob.Upload(context.Background(), bytes.NewReader(nil), + azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + + // create an extra blob that represents an empty ADLS directory, which should never be picked up + _, err = containerURL.NewBlockBlobURL(adlsDirName+"/neverpickup").Upload(context.Background(), bytes.NewReader(nil), + azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, adlsDirName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(blobList)) + }) + + // turn off recursive, this time only top blobs should be transferred + raw.recursive = false + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + c.Assert(len(mockedRPC.transfers), chk.Not(chk.Equals), len(blobList)) + + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + c.Assert(strings.Contains(localRelativeFilePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + }) +} diff --git a/cmd/zt_sync_processor_test.go b/cmd/zt_sync_processor_test.go new file mode 100644 index 000000000..a7ea5964c --- /dev/null +++ b/cmd/zt_sync_processor_test.go @@ -0,0 +1,99 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "os" + "path/filepath" +) + +type syncProcessorSuite struct{} + +var _ = chk.Suite(&syncProcessorSuite{}) + +func (s *syncProcessorSuite) TestLocalDeleter(c *chk.C) { + // set up the local file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := "extraFile.txt" + scenarioHelper{}.generateFilesFromList(c, dstDirName, []string{dstFileName}) + + // construct the cooked input to simulate user input + cca := &cookedSyncCmdArgs{ + destination: dstDirName, + deleteDestination: common.EDeleteDestination.True(), + } + + // set up local deleter + deleter := newSyncLocalDeleteProcessor(cca) + + // validate that the file still exists + _, err := os.Stat(filepath.Join(dstDirName, dstFileName)) + c.Assert(err, chk.IsNil) + + // exercise the deleter + err = deleter.removeImmediately(storedObject{relativePath: dstFileName}) + c.Assert(err, chk.IsNil) + + // validate that the file no longer exists + _, err = os.Stat(filepath.Join(dstDirName, dstFileName)) + c.Assert(err, chk.NotNil) +} + +func (s *syncProcessorSuite) TestBlobDeleter(c *chk.C) { + bsu := getBSU() + blobName := "extraBlob.pdf" + + // set up the blob to delete + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + scenarioHelper{}.generateBlobs(c, containerURL, []string{blobName}) + + // validate that the blob exists + blobURL := containerURL.NewBlobURL(blobName) + _, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + + // construct the cooked input to simulate user input + rawContainerURL := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + parts := azblob.NewBlobURLParts(rawContainerURL) + cca := &cookedSyncCmdArgs{ + destination: containerURL.String(), + destinationSAS: parts.SAS.Encode(), + credentialInfo: common.CredentialInfo{CredentialType: common.ECredentialType.Anonymous()}, + deleteDestination: common.EDeleteDestination.True(), + } + + // set up the blob deleter + deleter, err := newSyncBlobDeleteProcessor(cca) + c.Assert(err, chk.IsNil) + + // exercise the deleter + err = deleter.removeImmediately(storedObject{relativePath: blobName}) + c.Assert(err, chk.IsNil) + + // validate that the blob was deleted + _, err = blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + c.Assert(err, chk.NotNil) +} diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go new file mode 100644 index 000000000..ab25b1459 --- /dev/null +++ b/cmd/zt_sync_upload_test.go @@ -0,0 +1,342 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "context" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "path/filepath" + "strings" +) + +// regular file->blob sync +func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { + bsu := getBSU() + + for _, srcFileName := range []string{"singlefileisbest", "打麻将.txt", "%4509%4254$85140&"} { + // set up the source as a single file + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := []string{srcFileName} + scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) + + // set up the destination container with a single blob + dstBlobName := srcFileName + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, []string{dstBlobName}) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dstBlobName) + raw := getDefaultRawInput(filepath.Join(srcDirName, srcFileName), rawBlobURLWithSAS.String()) + + // the blob was created after the file, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // recreate the file to have a later last modified time + scenarioHelper{}.generateFilesFromList(c, srcDirName, []string{srcFileName}) + mockedRPC.reset() + + // the file was created after the blob, so the sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + validateUploadTransfersAreScheduled(c, filepath.Join(srcDirName, srcFileName), + containerURL.NewBlobURL(dstBlobName).String(), []string{""}, mockedRPC) + }) + } +} + +// regular directory->container sync but destination is empty, so everything has to be transferred +func (s *cmdIntegrationSuite) TestSyncUploadWithEmptyDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(fileList)) + + // validate that the right transfers were sent + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) + + // turn off recursive, this time only top blobs should be transferred + raw.recursive = false + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + c.Assert(len(mockedRPC.transfers), chk.Not(chk.Equals), len(fileList)) + + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Source, srcDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + c.Assert(strings.Contains(localRelativeFilePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + }) +} + +// regular directory->container sync but destination is identical to the source, transfers are scheduled based on lmt +func (s *cmdIntegrationSuite) TestSyncUploadWithIdenticalDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up an the container with the exact same files, but later lmts + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer + scenarioHelper{}.generateBlobs(c, containerURL, fileList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // refresh the files' last modified time so that they are newer + scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) +} + +// regular container->directory sync where destination is missing some files from source, and also has some extra files +func (s *cmdIntegrationSuite) TestSyncUploadWithMismatchedDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up an the container with half of the files, but later lmts + // also add some extra blobs that are not present at the source + extraBlobs := []string{"extraFile1.pdf, extraFile2.txt"} + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + scenarioHelper{}.generateBlobs(c, containerURL, fileList[0:len(fileList)/2]) + scenarioHelper{}.generateBlobs(c, containerURL, extraBlobs) + expectedOutput := fileList[len(fileList)/2:] + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), expectedOutput, mockedRPC) + + // make sure the extra blobs were deleted + for _, blobName := range extraBlobs { + exists := scenarioHelper{}.blobExists(containerURL.NewBlobURL(blobName)) + c.Assert(exists, chk.Equals, false) + } + }) +} + +// include flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncUploadWithIncludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // add special files that we wish to include + filesToInclude := []string{"important.pdf", "includeSub/amazing.jpeg", "exactName"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // set up the destination as an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + raw.include = includeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) + }) +} + +// exclude flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncUploadWithExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // add special files that we wish to exclude + filesToExclude := []string{"notGood.pdf", "excludeSub/lame.jpeg", "exactName"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToExclude) + excludeString := "*.pdf;*.jpeg;exactName" + + // set up the destination as an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) +} + +// include and exclude flag can work together to limit the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncUploadWithIncludeAndExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // add special files that we wish to include + filesToInclude := []string{"important.pdf", "includeSub/amazing.jpeg"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // add special files that we wish to exclude + // note that the excluded files also match the include string + filesToExclude := []string{"sorry.pdf", "exclude/notGood.jpeg", "exactName", "sub/exactName"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToExclude) + excludeString := "so*;not*;exactName" + + // set up the destination as an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + raw.include = includeString + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) + }) +} + +// validate the bug fix for this scenario +func (s *cmdIntegrationSuite) TestSyncUploadWithMissingDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up the destination as an non-existent container + containerURL, containerName := getContainerURL(c, bsu) + + // validate that the container does not exist + _, err := containerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{}) + c.Assert(err, chk.NotNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + // error should not be nil, but the app should not crash either + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} diff --git a/cmd/zt_test.go b/cmd/zt_test.go new file mode 100644 index 000000000..9c469b692 --- /dev/null +++ b/cmd/zt_test.go @@ -0,0 +1,276 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/ioutil" + "net/url" + "os" + "runtime" + "strings" + "testing" + "time" + + chk "gopkg.in/check.v1" + + "math/rand" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +// Hookup to the testing framework +func Test(t *testing.T) { chk.TestingT(t) } + +type cmdIntegrationSuite struct{} + +var _ = chk.Suite(&cmdIntegrationSuite{}) +var ctx = context.Background() + +const ( + containerPrefix = "container" + blobPrefix = "blob" + blockBlobDefaultData = "AzCopy Random Test Data" +) + +// This function generates an entity name by concatenating the passed prefix, +// the name of the test requesting the entity name, and the minute, second, and nanoseconds of the call. +// This should make it easy to associate the entities with their test, uniquely identify +// them, and determine the order in which they were created. +// Note that this imposes a restriction on the length of test names +func generateName(prefix string) string { + // These next lines up through the for loop are obtaining and walking up the stack + // trace to extrat the test name, which is stored in name + pc := make([]uintptr, 10) + runtime.Callers(0, pc) + f := runtime.FuncForPC(pc[0]) + name := f.Name() + for i := 0; !strings.Contains(name, "Suite"); i++ { // The tests are all scoped to the suite, so this ensures getting the actual test name + f = runtime.FuncForPC(pc[i]) + name = f.Name() + } + funcNameStart := strings.Index(name, "Test") + name = name[funcNameStart+len("Test"):] // Just get the name of the test and not any of the garbage at the beginning + name = strings.ToLower(name) // Ensure it is a valid resource name + currentTime := time.Now() + name = fmt.Sprintf("%s%s%d%d%d", prefix, strings.ToLower(name), currentTime.Minute(), currentTime.Second(), currentTime.Nanosecond()) + return name +} + +func generateContainerName() string { + return generateName(containerPrefix) +} + +func generateBlobName() string { + return generateName(blobPrefix) +} + +func getContainerURL(c *chk.C, bsu azblob.ServiceURL) (container azblob.ContainerURL, name string) { + name = generateContainerName() + container = bsu.NewContainerURL(name) + + return container, name +} + +func getBlockBlobURL(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.BlockBlobURL, name string) { + name = prefix + generateBlobName() + blob = container.NewBlockBlobURL(name) + + return blob, name +} + +func getAppendBlobURL(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.AppendBlobURL, name string) { + name = generateBlobName() + blob = container.NewAppendBlobURL(prefix + name) + + return blob, name +} + +func getPageBlobURL(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.PageBlobURL, name string) { + name = generateBlobName() + blob = container.NewPageBlobURL(prefix + name) + + return +} + +func getReaderToRandomBytes(n int) *bytes.Reader { + r, _ := getRandomDataAndReader(n) + return r +} + +func getRandomDataAndReader(n int) (*bytes.Reader, []byte) { + data := make([]byte, n, n) + rand.Read(data) + return bytes.NewReader(data), data +} + +func createNewContainer(c *chk.C, bsu azblob.ServiceURL) (container azblob.ContainerURL, name string) { + container, name = getContainerURL(c, bsu) + + cResp, err := container.Create(ctx, nil, azblob.PublicAccessNone) + c.Assert(err, chk.IsNil) + c.Assert(cResp.StatusCode(), chk.Equals, 201) + return container, name +} + +func createNewBlockBlob(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.BlockBlobURL, name string) { + blob, name = getBlockBlobURL(c, container, prefix) + + cResp, err := blob.Upload(ctx, strings.NewReader(blockBlobDefaultData), azblob.BlobHTTPHeaders{}, + nil, azblob.BlobAccessConditions{}) + + c.Assert(err, chk.IsNil) + c.Assert(cResp.StatusCode(), chk.Equals, 201) + + return +} + +func createNewAppendBlob(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.AppendBlobURL, name string) { + blob, name = getAppendBlobURL(c, container, prefix) + + resp, err := blob.Create(ctx, azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{}) + + c.Assert(err, chk.IsNil) + c.Assert(resp.StatusCode(), chk.Equals, 201) + return +} + +func createNewPageBlob(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.PageBlobURL, name string) { + blob, name = getPageBlobURL(c, container, prefix) + + resp, err := blob.Create(ctx, azblob.PageBlobPageBytes*10, 0, azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{}) + + c.Assert(err, chk.IsNil) + c.Assert(resp.StatusCode(), chk.Equals, 201) + return +} + +func deleteContainer(c *chk.C, container azblob.ContainerURL) { + resp, err := container.Delete(ctx, azblob.ContainerAccessConditions{}) + c.Assert(err, chk.IsNil) + c.Assert(resp.StatusCode(), chk.Equals, 202) +} + +func getGenericCredential(accountType string) (*azblob.SharedKeyCredential, error) { + accountNameEnvVar := accountType + "ACCOUNT_NAME" + accountKeyEnvVar := accountType + "ACCOUNT_KEY" + accountName, accountKey := os.Getenv(accountNameEnvVar), os.Getenv(accountKeyEnvVar) + if accountName == "" || accountKey == "" { + return nil, errors.New(accountNameEnvVar + " and/or " + accountKeyEnvVar + " environment variables not specified.") + } + return azblob.NewSharedKeyCredential(accountName, accountKey) +} + +func getGenericBSU(accountType string) (azblob.ServiceURL, error) { + credential, err := getGenericCredential(accountType) + if err != nil { + return azblob.ServiceURL{}, err + } + + pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + blobPrimaryURL, _ := url.Parse("https://" + credential.AccountName() + ".blob.core.windows.net/") + return azblob.NewServiceURL(*blobPrimaryURL, pipeline), nil +} + +func getBSU() azblob.ServiceURL { + bsu, _ := getGenericBSU("") + return bsu +} + +func validateStorageError(c *chk.C, err error, code azblob.ServiceCodeType) { + serr, _ := err.(azblob.StorageError) + c.Assert(serr.ServiceCode(), chk.Equals, code) +} + +func getRelativeTimeGMT(amount time.Duration) time.Time { + currentTime := time.Now().In(time.FixedZone("GMT", 0)) + currentTime = currentTime.Add(amount * time.Second) + return currentTime +} + +func generateCurrentTimeWithModerateResolution() time.Time { + highResolutionTime := time.Now().UTC() + return time.Date(highResolutionTime.Year(), highResolutionTime.Month(), highResolutionTime.Day(), highResolutionTime.Hour(), highResolutionTime.Minute(), + highResolutionTime.Second(), 0, highResolutionTime.Location()) +} + +// Some tests require setting service properties. It can take up to 30 seconds for the new properties to be reflected across all FEs. +// We will enable the necessary property and try to run the test implementation. If it fails with an error that should be due to +// those changes not being reflected yet, we will wait 30 seconds and try the test again. If it fails this time for any reason, +// we fail the test. It is the responsibility of the the testImplFunc to determine which error string indicates the test should be retried. +// There can only be one such string. All errors that cannot be due to this detail should be asserted and not returned as an error string. +func runTestRequiringServiceProperties(c *chk.C, bsu azblob.ServiceURL, code string, + enableServicePropertyFunc func(*chk.C, azblob.ServiceURL), + testImplFunc func(*chk.C, azblob.ServiceURL) error, + disableServicePropertyFunc func(*chk.C, azblob.ServiceURL)) { + enableServicePropertyFunc(c, bsu) + defer disableServicePropertyFunc(c, bsu) + err := testImplFunc(c, bsu) + // We cannot assume that the error indicative of slow update will necessarily be a StorageError. As in ListBlobs. + if err != nil && err.Error() == code { + time.Sleep(time.Second * 30) + err = testImplFunc(c, bsu) + c.Assert(err, chk.IsNil) + } +} + +func enableSoftDelete(c *chk.C, bsu azblob.ServiceURL) { + days := int32(1) + _, err := bsu.SetProperties(ctx, azblob.StorageServiceProperties{DeleteRetentionPolicy: &azblob.RetentionPolicy{Enabled: true, Days: &days}}) + c.Assert(err, chk.IsNil) +} + +func disableSoftDelete(c *chk.C, bsu azblob.ServiceURL) { + _, err := bsu.SetProperties(ctx, azblob.StorageServiceProperties{DeleteRetentionPolicy: &azblob.RetentionPolicy{Enabled: false}}) + c.Assert(err, chk.IsNil) +} + +func validateUpload(c *chk.C, blobURL azblob.BlockBlobURL) { + resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false) + c.Assert(err, chk.IsNil) + data, _ := ioutil.ReadAll(resp.Response().Body) + c.Assert(data, chk.HasLen, 0) +} + +func getContainerURLWithSAS(c *chk.C, credential azblob.SharedKeyCredential, containerName string) azblob.ContainerURL { + sasQueryParams, err := azblob.BlobSASSignatureValues{ + Protocol: azblob.SASProtocolHTTPS, + ExpiryTime: time.Now().UTC().Add(48 * time.Hour), + ContainerName: containerName, + Permissions: azblob.ContainerSASPermissions{Read: true, Add: true, Write: true, Create: true, Delete: true, List: true}.String(), + }.NewSASQueryParameters(&credential) + c.Assert(err, chk.IsNil) + + // construct the url from scratch + qp := sasQueryParams.Encode() + rawURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s?%s", + credential.AccountName(), containerName, qp) + + // convert the raw url and validate it was parsed successfully + fullURL, err := url.Parse(rawURL) + c.Assert(err, chk.IsNil) + + // TODO perhaps we need a global default pipeline + return azblob.NewContainerURL(*fullURL, azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})) +} diff --git a/common/chunkStatusLogger.go b/common/chunkStatusLogger.go index 35969697c..8cb6a6bf3 100644 --- a/common/chunkStatusLogger.go +++ b/common/chunkStatusLogger.go @@ -25,35 +25,133 @@ import ( "fmt" "os" "path" + "sync/atomic" "time" ) +// Identifies a chunk. Always create with NewChunkID type ChunkID struct { Name string OffsetInFile int64 + + // What is this chunk's progress currently waiting on? + // Must be a pointer, because the ChunkID itself is a struct. + // When chunkID is passed around, copies are made, + // but because this is a pointer, all will point to the same + // value for waitReasonIndex (so when we change it, all will see the change) + waitReasonIndex *int32 + + // Like waitReasonIndex, but is effectively just a boolean to track whether we are done. + // Must be a pointer, for same reason that waitReasonIndex is. + // Can't be done just off waitReasonIndex because for downloads we actually + // tell the jptm we are done before the chunk has been flushed out to disk, so + // waitReasonIndex isn't yet ready to go to "Done" at that time. + completionNotifiedToJptm *int32 + + // TODO: it's a bit odd having two pointers in a struct like this. Review, maybe we should always work + // with pointers to chunk ids, with nocopy? If we do that, the two fields that are currently pointers + // can become non-pointers } -var EWaitReason = WaitReason(0) +func NewChunkID(name string, offsetInFile int64) ChunkID { + dummyWaitReasonIndex := int32(0) + zeroNotificationState := int32(0) + return ChunkID{ + Name: name, + OffsetInFile: offsetInFile, + waitReasonIndex: &dummyWaitReasonIndex, // must initialize, so don't get nil pointer on usage + completionNotifiedToJptm: &zeroNotificationState, + } +} -type WaitReason string +func (id ChunkID) SetCompletionNotificationSent() { + if atomic.SwapInt32(id.completionNotifiedToJptm, 1) != 0 { + panic("cannot complete the same chunk twice") + } +} -// statuses used by both upload and download -func (WaitReason) RAMToSchedule() WaitReason { return WaitReason("RAM") } // waiting for enough RAM to schedule the chunk -func (WaitReason) WorkerGR() WaitReason { return WaitReason("GR") } // waiting for a goroutine to start running our chunkfunc -func (WaitReason) Body() WaitReason { return WaitReason("Body") } // waiting to finish sending/receiving the BODY -func (WaitReason) Disk() WaitReason { return WaitReason("Disk") } // waiting on disk write to complete -func (WaitReason) ChunkDone() WaitReason { return WaitReason("Done") } // not waiting on anything. Chunk is done. -func (WaitReason) Cancelled() WaitReason { return WaitReason("Cancelled") } // transfer was cancelled. All chunks end with either Done or Cancelled. +var EWaitReason = WaitReason{0, ""} -// extra statuses used only by download -func (WaitReason) HeaderResponse() WaitReason { return WaitReason("Head") } // waiting to finish downloading the HEAD -func (WaitReason) BodyReReadDueToMem() WaitReason { return WaitReason("BodyReRead-LowRam") } //waiting to re-read the body after a forced-retry due to low RAM -func (WaitReason) BodyReReadDueToSpeed() WaitReason { return WaitReason("BodyReRead-TooSlow") } // waiting to re-read the body after a forced-retry due to a slow chunk read (without low RAM) -func (WaitReason) WriterChannel() WaitReason { return WaitReason("Writer") } // waiting for the writer routine, in chunkedFileWriter, to pick up this chunk -func (WaitReason) PriorChunk() WaitReason { return WaitReason("Prior") } // waiting on a prior chunk to arrive (before this one can be saved) +// WaitReason identifies the one thing that a given chunk is waiting on, at a given moment. +// Basically = state, phrased in terms of "the thing I'm waiting for" +type WaitReason struct { + index int32 + Name string +} + +// Head (below) has index between GB and Body, just so the ordering is numerical ascending during typical chunk lifetime for both upload and download +func (WaitReason) Nothing() WaitReason { return WaitReason{0, "Nothing"} } // not waiting for anything +func (WaitReason) RAMToSchedule() WaitReason { return WaitReason{1, "RAM"} } // waiting for enough RAM to schedule the chunk +func (WaitReason) WorkerGR() WaitReason { return WaitReason{2, "Worker"} } // waiting for a goroutine to start running our chunkfunc +func (WaitReason) HeaderResponse() WaitReason { return WaitReason{3, "Head"} } // waiting to finish downloading the HEAD +func (WaitReason) Body() WaitReason { return WaitReason{4, "Body"} } // waiting to finish sending/receiving the BODY +func (WaitReason) BodyReReadDueToMem() WaitReason { return WaitReason{5, "BodyReRead-LowRam"} } //waiting to re-read the body after a forced-retry due to low RAM +func (WaitReason) BodyReReadDueToSpeed() WaitReason { return WaitReason{6, "BodyReRead-TooSlow"} } // waiting to re-read the body after a forced-retry due to a slow chunk read (without low RAM) +func (WaitReason) Sorting() WaitReason { return WaitReason{7, "Sorting"} } // waiting for the writer routine, in chunkedFileWriter, to pick up this chunk and sort it into sequence +func (WaitReason) PriorChunk() WaitReason { return WaitReason{8, "Prior"} } // waiting on a prior chunk to arrive (before this one can be saved) +func (WaitReason) QueueToWrite() WaitReason { return WaitReason{9, "Queue"} } // prior chunk has arrived, but is not yet written out to disk +func (WaitReason) DiskIO() WaitReason { return WaitReason{10, "DiskIO"} } // waiting on disk read/write to complete +func (WaitReason) ChunkDone() WaitReason { return WaitReason{11, "Done"} } // not waiting on anything. Chunk is done. +func (WaitReason) Cancelled() WaitReason { return WaitReason{12, "Cancelled"} } // transfer was cancelled. All chunks end with either Done or Cancelled. + +// TODO: consider change the above so that they don't create new struct on every call? Is that necessary/useful? +// Note: reason it's not using the normal enum approach, where it only has a number, is to try to optimize +// the String method below, on the assumption that it will be called a lot. Is that a premature optimization? + +// Upload chunks go through these states, in this order. +// We record this set of states, in this order, so that when we are uploading GetCounts() can return +// counts for only those states that are relevant to upload (some are not relevant, so they are not in this list) +// AND so that GetCounts will return the counts in the order that the states actually happen when uploading. +// That makes it easy for end-users of the counts (i.e. logging and display code) to show the state counts +// in a meaningful left-to-right sequential order. +var uploadWaitReasons = []WaitReason{ + // These first two happen in the transfer initiation function (i.e. the chunkfunc creation loop) + // So their total is constrained to the size of the goroutine pool that runs those functions. + // (e.g. 64, given the GR pool sizing as at Feb 2019) + EWaitReason.RAMToSchedule(), + EWaitReason.DiskIO(), + + // This next one is used when waiting for a worker Go routine to pick up the scheduled chunk func. + // Chunks in this state are effectively a queue of work waiting to be sent over the network + EWaitReason.WorkerGR(), + + // This is the actual network activity + EWaitReason.Body(), // header is not separated out for uploads, so is implicitly included here + // Plus Done/cancelled, which are not included here because not wanted for GetCounts +} + +// Download chunks go through a larger set of states, due to needing to be re-assembled into sequential order +// See comment on uploadWaitReasons for rationale. +var downloadWaitReasons = []WaitReason{ + // Done by the transfer initiation function (i.e. chunkfunc creation loop) + EWaitReason.RAMToSchedule(), + + // Waiting for a work Goroutine to pick up the chunkfunc and execute it. + // Chunks in this state are effectively a queue of work, waiting for their network downloads to be initiated + EWaitReason.WorkerGR(), + + // These next ones are the actual network activity + EWaitReason.HeaderResponse(), + EWaitReason.Body(), + // next two exist, but are not reported on separately in GetCounts, so are commented out + //EWaitReason.BodyReReadDueToMem(), + //EWaitReason.BodyReReadDueToSpeed(), + + // Sorting and QueueToWrite together comprise a queue of work waiting to be written to disk. + // The former are unsorted, and the latter have been sorted into sequential order. + // PriorChunk is unusual, because chunks in that wait state are not (yet) waiting for their turn to be written to disk, + // instead they are waiting on some prior chunk to finish arriving over the network + EWaitReason.Sorting(), + EWaitReason.PriorChunk(), + EWaitReason.QueueToWrite(), + + // The actual disk write + EWaitReason.DiskIO(), + // Plus Done/cancelled, which are not included here because not wanted for GetCounts +} func (wr WaitReason) String() string { - return string(wr) // avoiding reflection here, for speed, since will be called a lot + return string(wr.Name) // avoiding reflection here, for speed, since will be called a lot } type ChunkStatusLogger interface { @@ -62,34 +160,54 @@ type ChunkStatusLogger interface { type ChunkStatusLoggerCloser interface { ChunkStatusLogger + GetCounts(isDownload bool) []chunkStatusCount + IsDiskConstrained(isUpload, isDownload bool) bool CloseLog() } +// chunkStatusLogger records all chunk state transitions, and makes aggregate data immediately available +// for performance diagnostics. Also optionally logs every individual transition to a file. type chunkStatusLogger struct { - enabled bool + counts []int64 + outputEnabled bool unsavedEntries chan chunkWaitState } -func NewChunkStatusLogger(jobID JobID, logFileFolder string, enable bool) ChunkStatusLoggerCloser { +func NewChunkStatusLogger(jobID JobID, logFileFolder string, enableOutput bool) ChunkStatusLoggerCloser { logger := &chunkStatusLogger{ - enabled: enable, + counts: make([]int64, numWaitReasons()), + outputEnabled: enableOutput, unsavedEntries: make(chan chunkWaitState, 1000000), } - if enable { + if enableOutput { chunkLogPath := path.Join(logFileFolder, jobID.String()+"-chunks.log") // its a CSV, but using log extension for consistency with other files in the directory go logger.main(chunkLogPath) } return logger } +func numWaitReasons() int32 { + return EWaitReason.Cancelled().index + 1 // assume this is the last wait reason +} + +type chunkStatusCount struct { + WaitReason WaitReason + Count int64 +} + type chunkWaitState struct { ChunkID reason WaitReason waitStart time.Time } +//////////////////////////////////// basic functionality ////////////////////////////////// + func (csl *chunkStatusLogger) LogChunkStatus(id ChunkID, reason WaitReason) { - if !csl.enabled { + // always update the in-memory stats, even if output is disabled + csl.countStateTransition(id, reason) + + if !csl.outputEnabled { return } defer func() { @@ -103,7 +221,7 @@ func (csl *chunkStatusLogger) LogChunkStatus(id ChunkID, reason WaitReason) { } func (csl *chunkStatusLogger) CloseLog() { - if !csl.enabled { + if !csl.outputEnabled { return } close(csl.unsavedEntries) @@ -129,6 +247,107 @@ func (csl *chunkStatusLogger) main(chunkLogPath string) { } } +////////////////////////////// aggregate count and analysis support ////////////////////// + +// We maintain running totals of how many chunks are in each state. +// To do so, we must determine the new state (which is simply a parameter) and the old state. +// We obtain and track the old state within the chunkID itself. The alternative, of having a threadsafe +// map in the chunkStatusLogger, to track and look up the states, is considered a risk for performance. +func (csl *chunkStatusLogger) countStateTransition(id ChunkID, newReason WaitReason) { + + // Flip the chunk's state to indicate the new thing that it's waiting for now + oldReasonIndex := atomic.SwapInt32(id.waitReasonIndex, newReason.index) + + // Update the counts + // There's no need to lock the array itself. Instead just do atomic operations on the contents. + // (See https://groups.google.com/forum/#!topic/Golang-nuts/Ud4Dqin2Shc) + if oldReasonIndex > 0 && oldReasonIndex < int32(len(csl.counts)) { + atomic.AddInt64(&csl.counts[oldReasonIndex], -1) + } + if newReason.index < int32(len(csl.counts)) { + atomic.AddInt64(&csl.counts[newReason.index], 1) + } +} + +func (csl *chunkStatusLogger) getCount(reason WaitReason) int64 { + return atomic.LoadInt64(&csl.counts[reason.index]) +} + +// Gets the current counts of chunks in each wait state +// Intended for performance diagnostics and reporting +func (csl *chunkStatusLogger) GetCounts(isDownload bool) []chunkStatusCount { + + var allReasons []WaitReason + if isDownload { + allReasons = downloadWaitReasons + } else { + allReasons = uploadWaitReasons + } + + result := make([]chunkStatusCount, len(allReasons)) + for i, reason := range allReasons { + count := csl.getCount(reason) + + // for simplicity in consuming the results, all the body read states are rolled into one here + if reason == EWaitReason.BodyReReadDueToSpeed() || reason == EWaitReason.BodyReReadDueToMem() { + panic("body re-reads should not be requested in counts. They get rolled into the main Body one") + } + if reason == EWaitReason.Body() { + count += csl.getCount(EWaitReason.BodyReReadDueToSpeed()) + count += csl.getCount(EWaitReason.BodyReReadDueToMem()) + } + + result[i] = chunkStatusCount{reason, count} + } + return result +} + +func (csl *chunkStatusLogger) IsDiskConstrained(isUpload, isDownload bool) bool { + if isUpload { + return csl.isUploadDiskConstrained() + } else if isDownload { + return csl.isDownloadDiskConstrained() + } else { + return false // it's neither upload nor download (e.g. S2S) + } +} + +// is disk the bottleneck in an upload? +func (csl *chunkStatusLogger) isUploadDiskConstrained() bool { + // If we are uploading, and there's almost nothing waiting to go out over the network, then + // probably the reason there's not much queued is that the disk is slow. + // BTW, we can't usefully look at any of the _earlier_ states, because they happen in the _generation_ of the chunk funcs + // (not the _execution_ and so their counts will just tend to equal that of the small goroutine pool that runs them). + // It might be convenient if we could compare TWO queue sizes here, as we do in isDownloadDiskConstrained, but unfortunately our + // Jan 2019 architecture only gives us ONE useful queue-like state when uploading, so we can't compare two. + const nearZeroQueueSize = 10 // TODO: is there any intelligent way to set this threshold? It's just an arbitrary guestimate of "small" at the moment + queueForNetworkIsSmall := csl.getCount(EWaitReason.WorkerGR()) < nearZeroQueueSize + + beforeGRWaitQueue := csl.getCount(EWaitReason.RAMToSchedule()) + csl.getCount(EWaitReason.DiskIO()) + areStillReadingDisk := beforeGRWaitQueue > 0 // size of queue for network is irrelevant if we are no longer actually reading disk files, and therefore no longer putting anything into the queue for network + + return areStillReadingDisk && queueForNetworkIsSmall +} + +// is disk the bottleneck in a download? +func (csl *chunkStatusLogger) isDownloadDiskConstrained() bool { + // See how many chunks are waiting on the disk. I.e. are queued before the actual disk state. + // Don't include the "PriorChunk" state, because that's not actually waiting on disk at all, it + // can mean waiting on network and/or waiting-on-Storage-Service. We don't know which. So we just exclude it from consideration. + chunksWaitingOnDisk := csl.getCount(EWaitReason.Sorting()) + csl.getCount(EWaitReason.QueueToWrite()) + + // i.e. are queued before the actual network states + chunksWaitingOnNetwork := csl.getCount(EWaitReason.WorkerGR()) + + // if we have way more stuff waiting on disk than on network, we can assume disk is the bottleneck + const activeDiskQThreshold = 10 + const bigDifference = 5 // TODO: review/tune the arbitrary constant here + return chunksWaitingOnDisk > activeDiskQThreshold && // this test is in case both are near zero, as they would be near the end of the job + chunksWaitingOnDisk > bigDifference*chunksWaitingOnNetwork +} + +///////////////////////////////////// Sample LinqPad query for manual analysis of chunklog ///////////////////////////////////// + /* LinqPad query used to analyze/visualize the CSV as is follows: Needs CSV driver for LinqPad to open the CSV - e.g. https://github.com/dobrou/CsvLINQPadDriver diff --git a/common/chunkedFileWriter.go b/common/chunkedFileWriter.go index 336a7ec68..e84405426 100644 --- a/common/chunkedFileWriter.go +++ b/common/chunkedFileWriter.go @@ -22,7 +22,9 @@ package common import ( "context" + "crypto/md5" "errors" + "hash" "io" "math" "sync/atomic" @@ -33,7 +35,7 @@ import ( type ChunkedFileWriter interface { // WaitToScheduleChunk blocks until enough RAM is available to handle the given chunk, then it - // "reserves" that amount of RAM in the CacheLimiter and returns. + // "reserves" that amount of RAM in the CacheLimiter and returns. WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error // EnqueueChunk hands the given chunkContents over to the ChunkedFileWriter, to be written to disk. @@ -42,11 +44,11 @@ type ChunkedFileWriter interface { // While any error may be returned immediately, errors are more likely to be returned later, on either a subsequent // call to this routine or on the final return to Flush. // After the chunk is written to disk, its reserved memory byte allocation is automatically subtracted from the CacheLimiter. - EnqueueChunk(ctx context.Context, retryForcer func(), id ChunkID, chunkSize int64, chunkContents io.Reader) error + EnqueueChunk(ctx context.Context, id ChunkID, chunkSize int64, chunkContents io.Reader, retryable bool) error // Flush will block until all the chunks have been written to disk. err will be non-nil if and only in any chunk failed to write. // Flush must be called exactly once, after all chunks have been enqueued with EnqueueChunk. - Flush(ctx context.Context) (md5Hash string, err error) + Flush(ctx context.Context) (md5HashOfFileAsWritten []byte, err error) // MaxRetryPerDownloadBody returns the maximum number of retries that will be done for the download of a single chunk body MaxRetryPerDownloadBody() int @@ -79,11 +81,14 @@ type chunkedFileWriter struct { creationTime time.Time // used for completion - successMd5 chan string // TODO: use this when we do MD5s + successMd5 chan []byte failureError chan error // controls body-read retries. Public so value can be shared with retryReader maxRetryPerDownloadBody int + + // how will hashes be validated? + md5ValidationOption HashValidationOption } type fileChunk struct { @@ -91,7 +96,7 @@ type fileChunk struct { data []byte } -func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheLimiter CacheLimiter, chunkLogger ChunkStatusLogger, file io.WriteCloser, numChunks uint32, maxBodyRetries int) ChunkedFileWriter { +func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheLimiter CacheLimiter, chunkLogger ChunkStatusLogger, file io.WriteCloser, numChunks uint32, maxBodyRetries int, md5ValidationOption HashValidationOption) ChunkedFileWriter { // Set max size for buffered channel. The upper limit here is believed to be generous, given worker routine drains it constantly. // Use num chunks in file if lower than the upper limit, to prevent allocating RAM for lots of large channel buffers when dealing with // very large numbers of very small files. @@ -102,11 +107,12 @@ func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheL slicePool: slicePool, cacheLimiter: cacheLimiter, chunkLogger: chunkLogger, - successMd5: make(chan string), + successMd5: make(chan []byte), failureError: make(chan error, 1), newUnorderedChunks: make(chan fileChunk, chanBufferSize), creationTime: time.Now(), maxRetryPerDownloadBody: maxBodyRetries, + md5ValidationOption: md5ValidationOption, } go w.workerRoutine(ctx) return w @@ -123,7 +129,7 @@ const maxDesirableActiveChunks = 20 // TODO: can we find a sensible way to remov // at the time of scheduling the chunk (which is when this routine should be called). // Is here, as method of this struct, for symmetry with the point where we remove it's count // from the cache limiter, which is also in this struct. -func (w *chunkedFileWriter) WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error{ +func (w *chunkedFileWriter) WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error { w.chunkLogger.LogChunkStatus(id, EWaitReason.RAMToSchedule()) err := w.cacheLimiter.WaitUntilAddBytes(ctx, chunkSize, w.shouldUseRelaxedRamThreshold) if err == nil { @@ -133,9 +139,18 @@ func (w *chunkedFileWriter) WaitToScheduleChunk(ctx context.Context, id ChunkID, } // Threadsafe method to enqueue a new chunk for processing -func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, retryForcer func(), id ChunkID, chunkSize int64, chunkContents io.Reader) error { +func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, id ChunkID, chunkSize int64, chunkContents io.Reader, retryable bool) error { + readDone := make(chan struct{}) - w.setupProgressMonitoring(readDone, id, chunkSize, retryForcer) + if retryable { + // if retryable == true, that tells us that closing the reader + // is a safe way to force this particular reader to retry. + // (Typically this means it forces the reader to make one iteration around its internal retry loop. + // Going around that loop is hidden to the normal Read code (unless it exceeds the retry count threshold)) + closer := chunkContents.(io.Closer).Close // do the type assertion now, so get panic if it's not compatible. If we left it to the last minute, then the type would only be verified on the rare occasions when retries are required + retryForcer := func() { _ = closer() } + w.setupProgressMonitoring(readDone, id, chunkSize, retryForcer) + } // read into a buffer buffer := w.slicePool.RentSlice(uint32(chunkSize)) @@ -146,7 +161,7 @@ func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, retryForcer func() } // enqueue it - w.chunkLogger.LogChunkStatus(id, EWaitReason.WriterChannel()) + w.chunkLogger.LogChunkStatus(id, EWaitReason.Sorting()) select { case err = <-w.failureError: if err != nil { @@ -160,8 +175,8 @@ func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, retryForcer func() } } -// Waits until all chunks have been flush to disk, then returns -func (w *chunkedFileWriter) Flush(ctx context.Context) (string, error) { +// Flush waits until all chunks have been flush to disk, then returns the MD5 has of the file's bytes-as-we-saved-them +func (w *chunkedFileWriter) Flush(ctx context.Context) ([]byte, error) { // let worker know that no more will be coming close(w.newUnorderedChunks) @@ -169,13 +184,13 @@ func (w *chunkedFileWriter) Flush(ctx context.Context) (string, error) { select { case err := <-w.failureError: if err != nil { - return "", err + return nil, err } - return "", ChunkWriterAlreadyFailed // channel returned nil because it was closed and empty + return nil, ChunkWriterAlreadyFailed // channel returned nil because it was closed and empty case <-ctx.Done(): - return "", ctx.Err() - case hashAsAtCompletion := <-w.successMd5: - return hashAsAtCompletion, nil + return nil, ctx.Err() + case md5AtCompletion := <-w.successMd5: + return md5AtCompletion, nil } } @@ -191,6 +206,11 @@ func (w *chunkedFileWriter) MaxRetryPerDownloadBody() int { func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { nextOffsetToSave := int64(0) unsavedChunksByFileOffset := make(map[int64]fileChunk) + md5Hasher := md5.New() + if w.md5ValidationOption == EHashValidationOption.NoCheck() { + // save CPU time by not even computing a hash, if we are not going to check it + md5Hasher = &nullHasher{} + } for { var newChunk fileChunk @@ -202,8 +222,8 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { if !channelIsOpen { // If channel is closed, we know that flush as been called and we have read everything // So we are finished - // TODO: add returning of MD5 hash in the next line - w.successMd5 <- "" // everything is done. We know there was no error, because if there was an error we would have returned before now + // We know there was no error, because if there was an error we would have returned before now + w.successMd5 <- md5Hasher.Sum(nil) return } case <-ctx.Done(): @@ -217,7 +237,8 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { w.chunkLogger.LogChunkStatus(newChunk.id, EWaitReason.PriorChunk()) // may have to wait on prior chunks to arrive // Process all chunks that we can - err := w.saveAvailableChunks(unsavedChunksByFileOffset, &nextOffsetToSave) + w.setStatusForContiguousAvailableChunks(unsavedChunksByFileOffset, nextOffsetToSave) // update states of those that have all their prior ones already here + err := w.sequentiallyProcessAvailableChunks(unsavedChunksByFileOffset, &nextOffsetToSave, md5Hasher) if err != nil { w.failureError <- err close(w.failureError) // must close because many goroutines may be calling the public methods, and all need to be able to tell there's been an error, even tho only one will get the actual error @@ -226,16 +247,21 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { } } -// Saves available chunks that are sequential from nextOffsetToSave. Stops and returns as soon as it hits +// Hashes and saves available chunks that are sequential from nextOffsetToSave. Stops and returns as soon as it hits // a gap (i.e. the position of a chunk that hasn't arrived yet) -func (w *chunkedFileWriter) saveAvailableChunks(unsavedChunksByFileOffset map[int64]fileChunk, nextOffsetToSave *int64) error { +func (w *chunkedFileWriter) sequentiallyProcessAvailableChunks(unsavedChunksByFileOffset map[int64]fileChunk, nextOffsetToSave *int64, md5Hasher hash.Hash) error { for { + // Look for next chunk in sequence nextChunkInSequence, exists := unsavedChunksByFileOffset[*nextOffsetToSave] if !exists { return nil //its not there yet. That's OK. } - *nextOffsetToSave += int64(len(nextChunkInSequence.data)) + *nextOffsetToSave += int64(len(nextChunkInSequence.data)) // update immediately so we won't forget! + + // Add it to the hash (must do so sequentially for MD5) + md5Hasher.Write(nextChunkInSequence.data) + // Save it err := w.saveOneChunk(nextChunkInSequence) if err != nil { return err @@ -243,6 +269,19 @@ func (w *chunkedFileWriter) saveAvailableChunks(unsavedChunksByFileOffset map[in } } +// Advances the status of chunks which are no longer waiting on missing predecessors, but are instead just waiting on +// us to get around to (sequentially) saving them +func (w *chunkedFileWriter) setStatusForContiguousAvailableChunks(unsavedChunksByFileOffset map[int64]fileChunk, nextOffsetToSave int64) { + for { + nextChunkInSequence, exists := unsavedChunksByFileOffset[nextOffsetToSave] + if !exists { + return //its not there yet, so no need to touch anything AFTER it. THEY are still waiting for prior chunk + } + nextOffsetToSave += int64(len(nextChunkInSequence.data)) + w.chunkLogger.LogChunkStatus(nextChunkInSequence.id, EWaitReason.QueueToWrite()) // we WILL write this. Just may have to write others before it + } +} + // Saves one chunk to its destination func (w *chunkedFileWriter) saveOneChunk(chunk fileChunk) error { defer func() { @@ -252,7 +291,7 @@ func (w *chunkedFileWriter) saveOneChunk(chunk fileChunk) error { w.chunkLogger.LogChunkStatus(chunk.id, EWaitReason.ChunkDone()) // this chunk is all finished }() - w.chunkLogger.LogChunkStatus(chunk.id, EWaitReason.Disk()) + w.chunkLogger.LogChunkStatus(chunk.id, EWaitReason.DiskIO()) _, err := w.file.Write(chunk.data) // unlike Read, Write must process ALL the data, or have an error. It can't return "early". if err != nil { return err @@ -269,7 +308,6 @@ func (w *chunkedFileWriter) shouldUseRelaxedRamThreshold() bool { return atomic.LoadInt32(&w.activeChunkCount) <= maxDesirableActiveChunks } - // Are we currently in a memory-constrained situation? func (w *chunkedFileWriter) haveMemoryPressure(chunkSize int64) bool { didAdd := w.cacheLimiter.TryAddBytes(chunkSize, w.shouldUseRelaxedRamThreshold()) @@ -285,7 +323,7 @@ func (w *chunkedFileWriter) haveMemoryPressure(chunkSize int64) bool { // By retrying the slow chunk, we usually get a fast read. func (w *chunkedFileWriter) setupProgressMonitoring(readDone chan struct{}, id ChunkID, chunkSize int64, retryForcer func()) { if retryForcer == nil { - panic("retryForcer is nil. This probably means that the request pipeline is not producing cancelable requests. I.e. it is not producing response bodies that implement RequestCanceller") + panic("retryForcer is nil") } start := time.Now() initialReceivedCount := atomic.LoadInt32(&w.totalReceivedChunkCount) diff --git a/common/emptyChunkReader.go b/common/emptyChunkReader.go index 6eba3ff13..455ff365c 100644 --- a/common/emptyChunkReader.go +++ b/common/emptyChunkReader.go @@ -22,6 +22,7 @@ package common import ( "errors" + "hash" "io" ) @@ -29,8 +30,8 @@ import ( type emptyChunkReader struct { } -func (cr *emptyChunkReader) TryBlockingPrefetch(fileReader io.ReaderAt) bool { - return true +func (cr *emptyChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { + return nil } func (cr *emptyChunkReader) Seek(offset int64, whence int) (int64, error) { @@ -59,3 +60,7 @@ func (cr *emptyChunkReader) HasPrefetchedEntirelyZeros() bool { func (cr *emptyChunkReader) Length() int64 { return 0 } + +func (cr *emptyChunkReader) WriteBufferTo(h hash.Hash) { + return // no content to write +} diff --git a/common/environment.go b/common/environment.go index e9b9eb37e..e8fd0e558 100644 --- a/common/environment.go +++ b/common/environment.go @@ -62,3 +62,10 @@ func (EnvironmentVariable) ProfileCPU() EnvironmentVariable { func (EnvironmentVariable) ProfileMemory() EnvironmentVariable { return EnvironmentVariable{Name: "AZCOPY_PROFILE_MEM"} } + +func (EnvironmentVariable) ShowPerfStates() EnvironmentVariable { + return EnvironmentVariable{ + Name: "AZCOPY_SHOW_PERF_STATES", + Description: "If set, to anything, on-screen output will include counts of chunks by state", + } +} diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 0e99575c5..32e4b81ba 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -98,6 +98,26 @@ type Status uint32 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +type DeleteDestination uint32 + +var EDeleteDestination = DeleteDestination(0) + +func (DeleteDestination) False() DeleteDestination { return DeleteDestination(0) } +func (DeleteDestination) Prompt() DeleteDestination { return DeleteDestination(1) } +func (DeleteDestination) True() DeleteDestination { return DeleteDestination(2) } + +func (dd *DeleteDestination) Parse(s string) error { + val, err := enum.Parse(reflect.TypeOf(dd), s, true) + if err == nil { + *dd = val.(DeleteDestination) + } + return err +} + +func (dd DeleteDestination) String() string { + return enum.StringInt(dd, reflect.TypeOf(dd)) +} + type OutputFormat uint32 var EOutputFormat = OutputFormat(0) @@ -114,6 +134,10 @@ func (of *OutputFormat) Parse(s string) error { return err } +func (of OutputFormat) String() string { + return enum.StringInt(of, reflect.TypeOf(of)) +} + var EExitCode = ExitCode(0) type ExitCode uint32 @@ -218,11 +242,37 @@ func (j *JobStatus) AtomicStore(newJobStatus JobStatus) { atomic.StoreUint32((*uint32)(j), uint32(newJobStatus)) } -func (JobStatus) InProgress() JobStatus { return JobStatus(0) } -func (JobStatus) Paused() JobStatus { return JobStatus(1) } -func (JobStatus) Cancelling() JobStatus { return JobStatus(2) } -func (JobStatus) Cancelled() JobStatus { return JobStatus(3) } -func (JobStatus) Completed() JobStatus { return JobStatus(4) } +func (j *JobStatus) EnhanceJobStatusInfo(skippedTransfers, failedTransfers, successfulTransfers bool) JobStatus { + if failedTransfers && skippedTransfers { + return EJobStatus.CompletedWithErrorsAndSkipped() + } else if failedTransfers { + if successfulTransfers { + return EJobStatus.CompletedWithErrors() + } else { + return EJobStatus.Failed() + } + } else if skippedTransfers { + return EJobStatus.CompletedWithSkipped() + } else { + return EJobStatus.Completed() + } +} + +func (j *JobStatus) IsJobDone() bool { + return *j == EJobStatus.Completed() || *j == EJobStatus.Cancelled() || *j == EJobStatus.CompletedWithSkipped() || + *j == EJobStatus.CompletedWithErrors() || *j == EJobStatus.CompletedWithErrorsAndSkipped() || + *j == EJobStatus.Failed() +} + +func (JobStatus) InProgress() JobStatus { return JobStatus(0) } +func (JobStatus) Paused() JobStatus { return JobStatus(1) } +func (JobStatus) Cancelling() JobStatus { return JobStatus(2) } +func (JobStatus) Cancelled() JobStatus { return JobStatus(3) } +func (JobStatus) Completed() JobStatus { return JobStatus(4) } +func (JobStatus) CompletedWithErrors() JobStatus { return JobStatus(5) } +func (JobStatus) CompletedWithSkipped() JobStatus { return JobStatus(6) } +func (JobStatus) CompletedWithErrorsAndSkipped() JobStatus { return JobStatus(7) } +func (JobStatus) Failed() JobStatus { return JobStatus(8) } func (js JobStatus) String() string { return enum.StringInt(js, reflect.TypeOf(js)) } @@ -251,6 +301,19 @@ func fromToValue(from Location, to Location) FromTo { return FromTo((FromTo(from) << 8) | FromTo(to)) } +func (l Location) IsRemote() bool { + switch l { + case ELocation.BlobFS(), ELocation.Blob(), ELocation.File(): + return true + case ELocation.Local(), ELocation.Pipe(): + return false + default: + panic("unexpected location, please specify if it is remote") + } + + return false +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// var EFromTo = FromTo(0) @@ -524,6 +587,55 @@ func (ct *CredentialType) Parse(s string) error { return err } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +var EHashValidationOption = HashValidationOption(0) + +var DefaultHashValidationOption = EHashValidationOption.FailIfDifferent() + +type HashValidationOption uint8 + +// FailIfDifferent says fail if hashes different, but NOT fail if saved hash is +// totally missing. This is a balance of convenience (for cases where no hash is saved) vs strictness +// (to validate strictly when one is present) +func (HashValidationOption) FailIfDifferent() HashValidationOption { return HashValidationOption(0) } + +// Do not check hashes at download time at all +func (HashValidationOption) NoCheck() HashValidationOption { return HashValidationOption(1) } + +// LogOnly means only log if missing or different, don't fail the transfer +func (HashValidationOption) LogOnly() HashValidationOption { return HashValidationOption(2) } + +// FailIfDifferentOrMissing is the strictest option, and useful for testing or validation in cases when +// we _know_ there should be a hash +func (HashValidationOption) FailIfDifferentOrMissing() HashValidationOption { + return HashValidationOption(3) +} + +func (hvo HashValidationOption) String() string { + return enum.StringInt(hvo, reflect.TypeOf(hvo)) +} + +func (hvo *HashValidationOption) Parse(s string) error { + val, err := enum.ParseInt(reflect.TypeOf(hvo), s, true, true) + if err == nil { + *hvo = val.(HashValidationOption) + } + return err +} + +func (hvo HashValidationOption) MarshalJSON() ([]byte, error) { + return json.Marshal(hvo.String()) +} + +func (hvo *HashValidationOption) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + return hvo.Parse(s) +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const ( DefaultBlockBlobBlockSize = 8 * 1024 * 1024 diff --git a/common/fe-ste-models_test.go b/common/fe-ste-models_test.go new file mode 100644 index 000000000..39e77f547 --- /dev/null +++ b/common/fe-ste-models_test.go @@ -0,0 +1,91 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common_test + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" +) + +type feSteModelsTestSuite struct{} + +var _ = chk.Suite(&feSteModelsTestSuite{}) + +func (s *feSteModelsTestSuite) TestEnhanceJobStatusInfo(c *chk.C) { + status := common.EJobStatus + + status = status.EnhanceJobStatusInfo(true, true, true) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithErrorsAndSkipped()) + + status = status.EnhanceJobStatusInfo(true, true, false) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithErrorsAndSkipped()) + + status = status.EnhanceJobStatusInfo(true, false, true) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithSkipped()) + + status = status.EnhanceJobStatusInfo(true, false, false) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithSkipped()) + + status = status.EnhanceJobStatusInfo(false, true, true) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithErrors()) + + status = status.EnhanceJobStatusInfo(false, true, false) + c.Assert(status, chk.Equals, common.EJobStatus.Failed()) + + status = status.EnhanceJobStatusInfo(false, false, true) + c.Assert(status, chk.Equals, common.EJobStatus.Completed()) + + // No-op if all are false + status = status.EnhanceJobStatusInfo(false, false, false) + c.Assert(status, chk.Equals, common.EJobStatus.Completed()) +} + +func (s *feSteModelsTestSuite) TestIsJobDone(c *chk.C) { + status := common.EJobStatus.InProgress() + c.Assert(status.IsJobDone(), chk.Equals, false) + + status = status.Paused() + c.Assert(status.IsJobDone(), chk.Equals, false) + + status = status.Cancelling() + c.Assert(status.IsJobDone(), chk.Equals, false) + + status = status.Cancelled() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.Completed() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithErrors() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithSkipped() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithErrors() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithErrorsAndSkipped() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.Failed() + c.Assert(status.IsJobDone(), chk.Equals, true) +} \ No newline at end of file diff --git a/common/lifecyleMgr.go b/common/lifecyleMgr.go index 3bb200336..6331a353e 100644 --- a/common/lifecyleMgr.go +++ b/common/lifecyleMgr.go @@ -19,6 +19,7 @@ var lcm = func() (lcmgr *lifecycleMgr) { msgQueue: make(chan outputMessage, 1000), progressCache: "", cancelChannel: make(chan os.Signal, 1), + outputFormat: EOutputFormat.Text(), // output text by default } // kick off the single routine that processes output @@ -33,45 +34,33 @@ var lcm = func() (lcmgr *lifecycleMgr) { // create a public interface so that consumers outside of this package can refer to the lifecycle manager // but they would not be able to instantiate one type LifecycleMgr interface { - Progress(string) // print on the same line over and over again, not allowed to float up + Init(OutputBuilder) // let the user know the job has started and initial information like log location + Progress(OutputBuilder) // print on the same line over and over again, not allowed to float up + Exit(OutputBuilder, ExitCode) // indicates successful execution exit after printing, allow user to specify exit code Info(string) // simple print, allowed to float up + Error(string) // indicates fatal error, exit after printing, exit code is always Failed (1) Prompt(string) string // ask the user a question(after erasing the progress), then return the response - Exit(string, ExitCode) // exit after printing - Error(string) // print to stderr SurrenderControl() // give up control, this should never return InitiateProgressReporting(WorkController, bool) // start writing progress with another routine GetEnvironmentVariable(EnvironmentVariable) string // get the environment variable or its default value + SetOutputFormat(OutputFormat) // change the output format of the entire application } func GetLifecycleMgr() LifecycleMgr { return lcm } -var eMessageType = outputMessageType(0) - -// outputMessageType defines the nature of the output, ex: progress report, job summary, or error -type outputMessageType uint8 - -func (outputMessageType) Progress() outputMessageType { return outputMessageType(0) } // should be printed on the same line over and over again, not allowed to float up -func (outputMessageType) Info() outputMessageType { return outputMessageType(1) } // simple print, allowed to float up -func (outputMessageType) Exit() outputMessageType { return outputMessageType(2) } // exit after printing -func (outputMessageType) Prompt() outputMessageType { return outputMessageType(3) } // ask the user a question after erasing the progress -func (outputMessageType) Error() outputMessageType { return outputMessageType(4) } // print to stderr - -// defines the output and how it should be handled -type outputMessage struct { - msgContent string - msgType outputMessageType - exitCode ExitCode // only for when the application is meant to exit after printing (i.e. Error or Final) - inputChannel chan<- string // support getting a response from the user -} - // single point of control for all outputs type lifecycleMgr struct { msgQueue chan outputMessage progressCache string // useful for keeping job progress on the last line cancelChannel chan os.Signal waitEverCalled int32 + outputFormat OutputFormat +} + +func (lcm *lifecycleMgr) SetOutputFormat(format OutputFormat) { + lcm.outputFormat = format } func (lcm *lifecycleMgr) checkAndStartCPUProfiling() { @@ -84,10 +73,10 @@ func (lcm *lifecycleMgr) checkAndStartCPUProfiling() { lcm.Info(fmt.Sprintf("pprof start CPU profiling, and saving profiling data to: %q", cpuProfilePath)) f, err := os.Create(cpuProfilePath) if err != nil { - lcm.Exit(fmt.Sprintf("Fail to create file for CPU profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to create file for CPU profiling, %v", err)) } if err := pprof.StartCPUProfile(f); err != nil { - lcm.Exit(fmt.Sprintf("Fail to start CPU profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to start CPU profiling, %v", err)) } } } @@ -107,11 +96,11 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() { lcm.Info(fmt.Sprintf("pprof start memory profiling, and saving profiling data to: %q", memProfilePath)) f, err := os.Create(memProfilePath) if err != nil { - lcm.Exit(fmt.Sprintf("Fail to create file for memory profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to create file for memory profiling, %v", err)) } runtime.GC() if err := pprof.WriteHeapProfile(f); err != nil { - lcm.Exit(fmt.Sprintf("Fail to start memory profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to start memory profiling, %v", err)) } if err := f.Close(); err != nil { lcm.Info(fmt.Sprintf("Fail to close memory profiling file, %v", err)) @@ -119,24 +108,29 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() { } } -func (lcm *lifecycleMgr) Progress(msg string) { +func (lcm *lifecycleMgr) Init(o OutputBuilder) { lcm.msgQueue <- outputMessage{ - msgContent: msg, - msgType: eMessageType.Progress(), + msgContent: o(lcm.outputFormat), + msgType: eOutputMessageType.Init(), } } -func (lcm *lifecycleMgr) Info(msg string) { +func (lcm *lifecycleMgr) Progress(o OutputBuilder) { + messageContent := "" + if o != nil { + messageContent = o(lcm.outputFormat) + } + lcm.msgQueue <- outputMessage{ - msgContent: msg, - msgType: eMessageType.Info(), + msgContent: messageContent, + msgType: eOutputMessageType.Progress(), } } -func (lcm *lifecycleMgr) Error(msg string) { +func (lcm *lifecycleMgr) Info(msg string) { lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Error(), + msgType: eOutputMessageType.Info(), } } @@ -144,7 +138,7 @@ func (lcm *lifecycleMgr) Prompt(msg string) string { expectedInputChannel := make(chan string, 1) lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Prompt(), + msgType: eOutputMessageType.Prompt(), inputChannel: expectedInputChannel, } @@ -152,7 +146,8 @@ func (lcm *lifecycleMgr) Prompt(msg string) string { return <-expectedInputChannel } -func (lcm *lifecycleMgr) Exit(msg string, exitCode ExitCode) { +// TODO minor: consider merging with Exit +func (lcm *lifecycleMgr) Error(msg string) { // Check if need to do memory profiling, and do memory profiling accordingly before azcopy exits. lcm.checkAndTriggerMemoryProfiling() @@ -161,7 +156,29 @@ func (lcm *lifecycleMgr) Exit(msg string, exitCode ExitCode) { lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Exit(), + msgType: eOutputMessageType.Error(), + exitCode: EExitCode.Error(), + } + + // stall forever until the success message is printed and program exits + lcm.SurrenderControl() +} + +func (lcm *lifecycleMgr) Exit(o OutputBuilder, exitCode ExitCode) { + // Check if need to do memory profiling, and do memory profiling accordingly before azcopy exits. + lcm.checkAndTriggerMemoryProfiling() + + // Check if there is ongoing CPU profiling, and stop CPU profiling. + lcm.checkAndStopCPUProfiling() + + messageContent := "" + if o != nil { + messageContent = o(lcm.outputFormat) + } + + lcm.msgQueue <- outputMessage{ + msgContent: messageContent, + msgType: eOutputMessageType.Exit(), exitCode: exitCode, } @@ -176,6 +193,54 @@ func (lcm *lifecycleMgr) SurrenderControl() { } func (lcm *lifecycleMgr) processOutputMessage() { + // this function constantly pulls out message to output + // and pass them onto the right handler based on the output format + for { + switch msgToPrint := <-lcm.msgQueue; lcm.outputFormat { + case EOutputFormat.Json(): + lcm.processJSONOutput(msgToPrint) + case EOutputFormat.Text(): + lcm.processTextOutput(msgToPrint) + case EOutputFormat.None(): + lcm.processNoneOutput(msgToPrint) + default: + panic("unimplemented output format") + } + } +} + +func (lcm *lifecycleMgr) processNoneOutput(msgToOutput outputMessage) { + if msgToOutput.msgType == eOutputMessageType.Exit() { + os.Exit(int(msgToOutput.exitCode)) + } else if msgToOutput.msgType == eOutputMessageType.Error() { + os.Exit(int(EExitCode.Error())) + } + + // ignore all other outputs + return +} + +func (lcm *lifecycleMgr) processJSONOutput(msgToOutput outputMessage) { + msgType := msgToOutput.msgType + + // right now, we return nothing so that the default behavior is triggered for the part that intended to get response + if msgType == eOutputMessageType.Prompt() { + // TODO determine how prompts work with JSON output + msgToOutput.inputChannel <- "" + return + } + + // simply output the json message + // we assume the msgContent is already formatted correctly + fmt.Println(GetJsonStringFromTemplate(newJsonOutputTemplate(msgType, msgToOutput.msgContent))) + + // exit if needed + if msgType == eOutputMessageType.Exit() || msgType == eOutputMessageType.Error() { + os.Exit(int(msgToOutput.exitCode)) + } +} + +func (lcm *lifecycleMgr) processTextOutput(msgToOutput outputMessage) { // when a new line needs to overwrite the current line completely // we need to make sure that if the new line is shorter, we properly erase everything from the current line var matchLengthWithSpaces = func(curLineLength, newLineLength int) { @@ -186,80 +251,57 @@ func (lcm *lifecycleMgr) processOutputMessage() { } } - // NOTE: fmt.printf is being avoided on purpose (for memory optimization) - for { - switch msgToPrint := <-lcm.msgQueue; msgToPrint.msgType { - case eMessageType.Exit(): - // simply print and quit - // if no message is intended, avoid adding new lines - if msgToPrint.msgContent != "" { - fmt.Println("\n" + msgToPrint.msgContent) - } - os.Exit(int(msgToPrint.exitCode)) - - case eMessageType.Progress(): - fmt.Print("\r") // return carriage back to start - fmt.Print(msgToPrint.msgContent) // print new progress - - // it is possible that the new progress status is somehow shorter than the previous one - // in this case we must erase the left over characters from the previous progress - matchLengthWithSpaces(len(lcm.progressCache), len(msgToPrint.msgContent)) - - lcm.progressCache = msgToPrint.msgContent - - case eMessageType.Info(): - if lcm.progressCache != "" { // a progress status is already on the last line - // print the info from the beginning on current line - fmt.Print("\r") - fmt.Print(msgToPrint.msgContent) + switch msgToOutput.msgType { + case eOutputMessageType.Error(), eOutputMessageType.Exit(): + // simply print and quit + // if no message is intended, avoid adding new lines + if msgToOutput.msgContent != "" { + fmt.Println("\n" + msgToOutput.msgContent) + } + os.Exit(int(msgToOutput.exitCode)) - // it is possible that the info is shorter than the progress status - // in this case we must erase the left over characters from the progress status - matchLengthWithSpaces(len(lcm.progressCache), len(msgToPrint.msgContent)) + case eOutputMessageType.Progress(): + fmt.Print("\r") // return carriage back to start + fmt.Print(msgToOutput.msgContent) // print new progress - // print the previous progress status again, so that it's on the last line - fmt.Print("\n") - fmt.Print(lcm.progressCache) - } else { - fmt.Println(msgToPrint.msgContent) - } + // it is possible that the new progress status is somehow shorter than the previous one + // in this case we must erase the left over characters from the previous progress + matchLengthWithSpaces(len(lcm.progressCache), len(msgToOutput.msgContent)) - case eMessageType.Error(): - // we need to print to stderr but it's mostly likely that both stdout and stderr are directed to the terminal - // in case we are already printing progress to stdout, we need to make sure that the content from - // stderr gets displayed properly on its own line - if lcm.progressCache != "" { // a progress status is already on the last line - // erase the progress status - fmt.Print("\r") - matchLengthWithSpaces(len(lcm.progressCache), 0) - fmt.Print("\r") - - os.Stderr.WriteString(msgToPrint.msgContent) - - // print the previous progress status again, so that it's on the last line - fmt.Print("\n") - fmt.Print(lcm.progressCache) - } else { - os.Stderr.WriteString(msgToPrint.msgContent) - } + lcm.progressCache = msgToOutput.msgContent - case eMessageType.Prompt(): - if lcm.progressCache != "" { // a progress status is already on the last line - // print the prompt from the beginning on current line - fmt.Print("\r") - fmt.Print(msgToPrint.msgContent) + case eOutputMessageType.Init(), eOutputMessageType.Info(): + if lcm.progressCache != "" { // a progress status is already on the last line + // print the info from the beginning on current line + fmt.Print("\r") + fmt.Print(msgToOutput.msgContent) - // it is possible that the prompt is shorter than the progress status - // in this case we must erase the left over characters from the progress status - matchLengthWithSpaces(len(lcm.progressCache), len(msgToPrint.msgContent)) + // it is possible that the info is shorter than the progress status + // in this case we must erase the left over characters from the progress status + matchLengthWithSpaces(len(lcm.progressCache), len(msgToOutput.msgContent)) - } else { - fmt.Print(msgToPrint.msgContent) - } - - // read the response to the prompt and send it back through the channel - msgToPrint.inputChannel <- lcm.readInCleanLineFromStdIn() + // print the previous progress status again, so that it's on the last line + fmt.Print("\n") + fmt.Print(lcm.progressCache) + } else { + fmt.Println(msgToOutput.msgContent) + } + case eOutputMessageType.Prompt(): + if lcm.progressCache != "" { // a progress status is already on the last line + // print the prompt from the beginning on current line + fmt.Print("\r") + fmt.Print(msgToOutput.msgContent) + + // it is possible that the prompt is shorter than the progress status + // in this case we must erase the left over characters from the progress status + matchLengthWithSpaces(len(lcm.progressCache), len(msgToOutput.msgContent)) + + } else { + fmt.Print(msgToOutput.msgContent) } + + // read the response to the prompt and send it back through the channel + msgToOutput.inputChannel <- lcm.readInCleanLineFromStdIn() } } diff --git a/common/logger.go b/common/logger.go index be4751072..eeba30175 100644 --- a/common/logger.go +++ b/common/logger.go @@ -21,11 +21,13 @@ package common import ( + "fmt" "log" + "net/url" "os" - "runtime" - "path" + "runtime" + "strings" "github.com/Azure/azure-pipeline-go/pipeline" ) @@ -129,7 +131,7 @@ func (jl *jobLogger) OpenLog() { jl.file = file jl.logger = log.New(jl.file, "", log.LstdFlags|log.LUTC) // Log the Azcopy Version - jl.logger.Println("AzcopVersion ", AzcopyVersion) + jl.logger.Println("AzcopyVersion ", AzcopyVersion) // Log the OS Environment and OS Architecture jl.logger.Println("OS-Environment ", runtime.GOOS) jl.logger.Println("OS-Architecture ", runtime.GOARCH) @@ -155,6 +157,12 @@ func (jl *jobLogger) CloseLog() { func (jl jobLogger) Log(loglevel pipeline.LogLevel, msg string) { // If the logger for Job is not initialized i.e file is not open // or logger instance is not initialized, then initialize it + + // Go, and therefore the sdk, defaults to \n for line endings, so if the platform has a different line ending, + // we should replace them to ensure readability on the given platform. + if lineEnding != "\n" { + msg = strings.Replace(msg, "\n", lineEnding, -1) + } if jl.ShouldLog(loglevel) { jl.logger.Println(msg) } @@ -165,3 +173,31 @@ func (jl jobLogger) Panic(err error) { jl.appLogger.Panic(err) // We panic here that it logs and the app terminates // We should never reach this line of code! } + +const TryEquals string = "Try=" // TODO: refactor so that this can be used by the retry policies too? So that when you search the logs for Try= you are guaranteed to find both types of retry (i.e. request send retries, and body read retries) + +func NewReadLogFunc(logger ILogger, fullUrl *url.URL) func(int, error, int64, int64, bool) { + redactedUrl := URLStringExtension(fullUrl.String()).RedactSigQueryParamForLogging() + + return func(failureCount int, err error, offset int64, count int64, willRetry bool) { + retryMessage := "Will retry" + if !willRetry { + retryMessage = "Will NOT retry" + } + logger.Log(pipeline.LogInfo, fmt.Sprintf( + "Error reading body of reply. Next try (if any) will be %s%d. %s. Error: %s. Offset: %d Count: %d URL: %s", + TryEquals, // so that retry wording for body-read retries is similar to that for URL-hitting retries + + // We log the number of the NEXT try, not the failure just done, so that users searching the log for "Try=2" + // will find ALL retries, both the request send retries (which are logged as try 2 when they are made) and + // body read retries (for which only the failure is logged - so if we did the actual failure number, there would be + // not Try=2 in the logs if the retries work). + failureCount+1, + + retryMessage, + err, + offset, + count, + redactedUrl)) + } +} diff --git a/common/mmf_darwin.go b/common/mmf_darwin.go index 0d4b908af..522351985 100644 --- a/common/mmf_darwin.go +++ b/common/mmf_darwin.go @@ -28,6 +28,8 @@ import ( "syscall" ) +const lineEnding = "\n" + type MMF struct { // slice represents the actual memory mapped buffer slice []byte diff --git a/common/mmf_linux.go b/common/mmf_linux.go index 6dddd39c7..911e34506 100644 --- a/common/mmf_linux.go +++ b/common/mmf_linux.go @@ -28,6 +28,8 @@ import ( "syscall" ) +const lineEnding = "\n" + type MMF struct { // slice represents the actual memory mapped buffer slice []byte diff --git a/common/mmf_windows.go b/common/mmf_windows.go index 89d7b9b4a..bf1192a4e 100644 --- a/common/mmf_windows.go +++ b/common/mmf_windows.go @@ -28,6 +28,8 @@ import ( "unsafe" ) +const lineEnding = "\r\n" + type MMF struct { // slice represents the actual memory mapped buffer slice []byte diff --git a/common/multiSizeSlicePool.go b/common/multiSizeSlicePool.go index 5bee56fd4..f0e0a40c3 100644 --- a/common/multiSizeSlicePool.go +++ b/common/multiSizeSlicePool.go @@ -29,6 +29,7 @@ import ( type ByteSlicePooler interface { RentSlice(desiredLength uint32) []byte ReturnSlice(slice []byte) + Prune() } // Pools byte slices of a single size. @@ -86,26 +87,64 @@ func NewMultiSizeSlicePool(maxSliceLength uint32) ByteSlicePooler { maxSlotIndex, _ := getSlotInfo(maxSliceLength) poolsBySize := make([]*simpleSlicePool, maxSlotIndex+1) for i := 0; i <= maxSlotIndex; i++ { - poolsBySize[i] = newSimpleSlicePool(1000) // TODO: review capacity (setting too low doesn't break anything, since we don't block when full, so maybe only 100 or so is OK?) + maxCount := getMaxSliceCountInPool(i) + poolsBySize[i] = newSimpleSlicePool(maxCount) } return &multiSizeSlicePool{poolsBySize: poolsBySize} } +var indexOf32KSlot, _ = getSlotInfo(32 * 1024) + +// For a given requested len(slice), this returns the slot index to use, and the max +// cap(slice) of the slices that will be found at that index func getSlotInfo(exactSliceLength uint32) (slotIndex int, maxCapInSlot int) { - // slot index is fast computation of the base-2 logarithm, rounded down - slotIndex = 32 - bits.LeadingZeros32(exactSliceLength) - // max cap in slot is the biggest number that maps to that slot index - // (e.g. slot index of 1 (which=2 to the power of 0) is 1, so (2 to the power of slotIndex) - // is the first number that doesn't fit the slot) - maxCapInSlot = (1 << uint(slotIndex)) - 1 - - // check TODO: replace this check with a proper unit test - if 32-bits.LeadingZeros32(uint32(maxCapInSlot)) != slotIndex { - panic("cross check of cap and slot index failed") + if exactSliceLength <= 0 { + panic("exact slice length must be greater than zero") + } + // raw slot index is fast computation of the base-2 logarithm, rounded down... + rawSlotIndex := 31 - bits.LeadingZeros32(exactSliceLength) + + // ...but in most cases we actually want to round up. + // E.g. we want 255 to go into the same bucket as 256. Why? because we want exact + // powers of 2 to be the largest thing in each bucket, since usually + // we will be using powers of 2, and that means we will usually be using + // all the allocated capacity (i.e. len == cap). That gives the most efficient use of RAM. + // The only time we don't want to round up, is if we already had an exact power of + // 2 to start with. + isExactPowerOfTwo := bits.OnesCount32(exactSliceLength) == 1 + if isExactPowerOfTwo { + slotIndex = rawSlotIndex + } else { + slotIndex = rawSlotIndex + 1 } + + // Max cap in slot is the biggest number that maps to that slot index + // (e.g. slot index of exactSliceLength=1 (which=2 to the power of 0) + // is 0 (because log-base2 of 1 == 0), so (2 to the power of slotIndex) + // is the highest number that still fits the slot) + maxCapInSlot = 1 << uint(slotIndex) + return } +func holdsSmallSlices(slotIndex int) bool { + return slotIndex <= indexOf32KSlot +} + +// For a given slot index, this returns the max number of pooled slices which the pool at +// that index should be allowed to hold. +func getMaxSliceCountInPool(slotIndex int) int { + if holdsSmallSlices(slotIndex) { + // Choose something fairly high for these because there's no significant RAM + // cost in doing so, and small files are a tricky case for perf so let's give + // them all the pooling help we can + return 500 + } else { + // Limit the medium and large ones a bit more strictly. + return 100 + } +} + // RentSlice borrows a slice from the pool (or creates a new one if none of suitable capacity is available) // Note that the returned slice may contain non-zero data - i.e. old data from the previous time it was used. // That's safe IFF you are going to do the likes of io.ReadFull to read into it, since you know that all of the @@ -146,3 +185,24 @@ func (mp *multiSizeSlicePool) ReturnSlice(slice []byte) { // put the slice back into the pool pool.Put(slice) } + +// Prune inactive stuff in all the big slots if due (don't worry about the little ones, they don't eat much RAM) +// Why do this? Because for the large slot sizes its hard to deal with slots that are full and IDLE. +// I.e. we were using them, but now we're working with other files in the same job that have different chunk sizes, +// so we have a pool slot that's full of slices that are no longer getting used. +// Would it work to just give the slot a very small max capacity? Maybe. E.g. max number in slot = 4 for the 128 MB slot. +// But can we be confident that 4 is enough for the pooling to have the desired benefits? Not sure. +// Hence the pruning, so that we don't need to set the fixed limits that low. With pruning, the count will (gradually) +// come down only when the slot is IDLE. +func (mp *multiSizeSlicePool) Prune() { + for index := 0; index < len(mp.poolsBySize); index++ { + shouldPrune := !holdsSmallSlices(index) + if shouldPrune { + // Get one item from the pool and throw it away. + // With repeated calls of Prune, this will gradually drain idle pools. + // But, since Prune is not called very often, + // it won't have much adverse impact on active pools. + _ = mp.poolsBySize[index].Get() + } + } +} diff --git a/common/nullHasher.go b/common/nullHasher.go new file mode 100644 index 000000000..a5a943d4b --- /dev/null +++ b/common/nullHasher.go @@ -0,0 +1,45 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common + +// A hash.Hash implementation that does nothing +type nullHasher struct{} + +func (*nullHasher) Write(p []byte) (n int, err error) { + // noop + return 0, nil +} + +func (*nullHasher) Sum(b []byte) []byte { + return make([]byte, 0) +} + +func (*nullHasher) Reset() { + // noop +} + +func (*nullHasher) Size() int { + return 0 +} + +func (*nullHasher) BlockSize() int { + return 1 +} diff --git a/common/output.go b/common/output.go new file mode 100644 index 000000000..ddfe52b4b --- /dev/null +++ b/common/output.go @@ -0,0 +1,80 @@ +package common + +import ( + "encoding/json" + "github.com/JeffreyRichter/enum/enum" + "reflect" + "strings" + "time" +) + +var eOutputMessageType = outputMessageType(0) + +// outputMessageType defines the nature of the output, ex: progress report, job summary, or error +type outputMessageType uint8 + +func (outputMessageType) Init() outputMessageType { return outputMessageType(0) } // simple print, allowed to float up +func (outputMessageType) Info() outputMessageType { return outputMessageType(1) } // simple print, allowed to float up +func (outputMessageType) Progress() outputMessageType { return outputMessageType(2) } // should be printed on the same line over and over again, not allowed to float up +func (outputMessageType) Exit() outputMessageType { return outputMessageType(3) } // exit after printing +func (outputMessageType) Error() outputMessageType { return outputMessageType(4) } // indicate fatal error, exit right after +func (outputMessageType) Prompt() outputMessageType { return outputMessageType(5) } // ask the user a question after erasing the progress + +func (o outputMessageType) String() string { + return enum.StringInt(o, reflect.TypeOf(o)) +} + +// defines the output and how it should be handled +type outputMessage struct { + msgContent string + msgType outputMessageType + exitCode ExitCode // only for when the application is meant to exit after printing (i.e. Error or Final) + inputChannel chan<- string // support getting a response from the user +} + +// used for output types that are not simple strings, such as progress and init +// a given format(text,json) is passed in, and the appropriate string is returned +type OutputBuilder func(OutputFormat) string + +// -------------------------------------- JSON templates -------------------------------------- // +// used to help formatting of JSON outputs + +func GetJsonStringFromTemplate(template interface{}) string { + jsonOutput, err := json.Marshal(template) + PanicIfErr(err) + + return string(jsonOutput) +} + +// defines the general output template when the format is set to json +type jsonOutputTemplate struct { + TimeStamp time.Time + MessageType string + MessageContent string // a simple string for INFO and ERROR, a serialized JSON for INIT, PROGRESS, EXIT +} + +func newJsonOutputTemplate(messageType outputMessageType, messageContent string) *jsonOutputTemplate { + return &jsonOutputTemplate{TimeStamp: time.Now(), MessageType: messageType.String(), MessageContent: messageContent} +} + +type InitMsgJsonTemplate struct { + LogFileLocation string + JobID string +} + +func GetStandardInitOutputBuilder(jobID string, logFileLocation string) OutputBuilder { + return func(format OutputFormat) string { + if format == EOutputFormat.Json() { + return GetJsonStringFromTemplate(InitMsgJsonTemplate{ + JobID: jobID, + LogFileLocation: logFileLocation, + }) + } + + var sb strings.Builder + sb.WriteString("\nJob " + jobID + " has started\n") + sb.WriteString("Log file is located at: " + logFileLocation) + sb.WriteString("\n") + return sb.String() + } +} diff --git a/common/rpc-models.go b/common/rpc-models.go index 4fed36879..38067cf7a 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -52,6 +52,8 @@ type CopyJobPartOrderRequest struct { Exclude map[string]int // list of blobTypes to exclude. ExcludeBlobType []azblob.BlobType + SourceRoot string + DestinationRoot string Transfers []CopyTransfer LogLevel LogLevel BlobAttributes BlobTransferAttributes @@ -69,31 +71,6 @@ type CredentialInfo struct { OAuthTokenInfo OAuthTokenInfo } -type SyncJobPartOrderRequest struct { - JobID JobID - FromTo FromTo - PartNumber PartNumber - LogLevel LogLevel - Include map[string]int - Exclude map[string]int - BlockSizeInBytes uint32 - SourceSAS string - DestinationSAS string - CopyJobRequest CopyJobPartOrderRequest - DeleteJobRequest CopyJobPartOrderRequest - // FilesDeletedLocally is used to keep track of the file that are deleted locally - // Since local files to delete are not sent as transfer to STE - // the count of the local files deletion is tracked using it. - FilesToDeleteLocally []string - // commandString hold the user given command which is logged to the Job log file - CommandString string - CredentialInfo CredentialInfo - - SourceFiles map[string]time.Time - - SourceFilesToExclude map[string]time.Time -} - type CopyJobPartOrderResponse struct { ErrorMsg string JobStarted bool @@ -108,14 +85,15 @@ type ListRequest struct { // This struct represents the optional attribute for blob request header type BlobTransferAttributes struct { - BlobType BlobType // The type of a blob - BlockBlob, PageBlob, AppendBlob - ContentType string //The content type specified for the blob. - ContentEncoding string //Specifies which content encodings have been applied to the blob. - BlockBlobTier BlockBlobTier // Specifies the tier to set on the block blobs. - PageBlobTier PageBlobTier // Specifies the tier to set on the page blobs. - Metadata string //User-defined Name-value pairs associated with the blob - NoGuessMimeType bool // represents user decision to interpret the content-encoding from source file - PreserveLastModifiedTime bool // when downloading, tell engine to set file's timestamp to timestamp of blob + BlobType BlobType // The type of a blob - BlockBlob, PageBlob, AppendBlob + ContentType string //The content type specified for the blob. + ContentEncoding string //Specifies which content encodings have been applied to the blob. + BlockBlobTier BlockBlobTier // Specifies the tier to set on the block blobs. + PageBlobTier PageBlobTier // Specifies the tier to set on the page blobs. + Metadata string //User-defined Name-value pairs associated with the blob + NoGuessMimeType bool // represents user decision to interpret the content-encoding from source file + PreserveLastModifiedTime bool // when downloading, tell engine to set file's timestamp to timestamp of blob + MD5ValidationOption HashValidationOption // when downloading, how strictly should we validate MD5 hashes? BlockSizeInBytes uint32 } @@ -139,8 +117,8 @@ type ListContainerResponse struct { // represents the JobProgressPercentage Summary response for list command when requested the Job Progress Summary for given JobId type ListJobSummaryResponse struct { ErrorMsg string - Timestamp time.Time - JobID JobID + Timestamp time.Time `json:"-"` + JobID JobID `json:"-"` // TODO: added for debugging purpose. remove later ActiveConnections int64 // CompleteJobOrdered determines whether the Job has been completely ordered or not @@ -157,13 +135,15 @@ type ListJobSummaryResponse struct { TotalBytesEnumerated uint64 FailedTransfers []TransferDetail SkippedTransfers []TransferDetail + IsDiskConstrained bool + PerfStrings []string `json:"-"` } // represents the JobProgressPercentage Summary response for list command when requested the Job Progress Summary for given JobId type ListSyncJobSummaryResponse struct { ErrorMsg string - Timestamp time.Time - JobID JobID + Timestamp time.Time `json:"-"` + JobID JobID `json:"-"` // TODO: added for debugging purpose. remove later ActiveConnections int64 // CompleteJobOrdered determines whether the Job has been completely ordered or not @@ -177,6 +157,12 @@ type ListSyncJobSummaryResponse struct { DeleteTransfersCompleted uint32 DeleteTransfersFailed uint32 FailedTransfers []TransferDetail + IsDiskConstrained bool + PerfStrings []string `json:"-"` + // sum of the size of transfer completed successfully so far. + TotalBytesTransferred uint64 + // sum of the total transfer enumerated so far. + TotalBytesEnumerated uint64 } type ListJobTransfersRequest struct { diff --git a/common/singleChunkReader.go b/common/singleChunkReader.go index eb42f0b9b..3ef939f12 100644 --- a/common/singleChunkReader.go +++ b/common/singleChunkReader.go @@ -21,9 +21,15 @@ package common import ( + "bytes" "context" "errors" + "github.com/Azure/azure-pipeline-go/pipeline" + "hash" "io" + "math" + "runtime" + "sync/atomic" ) // Reader of ONE chunk of a file. Maybe used to re-read multiple times (e.g. if @@ -42,11 +48,8 @@ type SingleChunkReader interface { // Closer is needed to clean up resources io.Closer - // TryBlockingPrefetch tries to read the full contents of the chunk into RAM. Returns true if succeeded for false if failed, - // although callers do not have to check the return value (and there is no error object returned). Why? - // Because its OK to keep using this object even if the prefetch fails, since in that case - // any subsequent Read will just retry the same read as we do here, and if it fails at that time then Read will return an error. - TryBlockingPrefetch(fileReader io.ReaderAt) bool + // BlockingPrefetch tries to read the full contents of the chunk into RAM. + BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error // CaptureLeadingBytes is used to grab enough of the initial bytes to do MIME-type detection. Expected to be called only // on the first chunk in each file (since there's no point in calling it on others) @@ -61,6 +64,10 @@ type SingleChunkReader interface { // In the rare edge case where this returns false due to the prefetch having failed (rather than the contents being non-zero), // we'll just treat it as a non-zero chunk. That's simpler (to code, to review and to test) than having this code force a prefetch. HasPrefetchedEntirelyZeros() bool + + // WriteBufferTo writes the entire contents of the prefetched buffer to h + // Panics if the internal buffer has not been prefetched (or if its been discarded after a complete Read) + WriteBufferTo(h hash.Hash) } // Simple aggregation of existing io interfaces @@ -73,6 +80,9 @@ type CloseableReaderAt interface { type ChunkReaderSourceFactory func() (CloseableReaderAt, error) type singleChunkReader struct { + // for diagnostics for issue https://github.com/Azure/azure-storage-azcopy/issues/191 + atomicUseIndicator int32 + // context used to allow cancellation of blocking operations // (Yes, ideally contexts are not stored in structs, but we need it inside Read, and there's no way for it to be passed in there) ctx context.Context @@ -86,6 +96,9 @@ type singleChunkReader struct { // for logging chunk state transitions chunkLogger ChunkStatusLogger + // general-purpose logger + generalLogger ILogger + // A factory to get hold of the file, in case we need to re-read any of it sourceFactory ChunkReaderSourceFactory @@ -103,13 +116,14 @@ type singleChunkReader struct { // TODO: pooling of buffers to reduce pressure on GC? } -func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFactory, chunkId ChunkID, length int64, chunkLogger ChunkStatusLogger, slicePool ByteSlicePooler, cacheLimiter CacheLimiter) SingleChunkReader { +func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFactory, chunkId ChunkID, length int64, chunkLogger ChunkStatusLogger, generalLogger ILogger, slicePool ByteSlicePooler, cacheLimiter CacheLimiter) SingleChunkReader { if length <= 0 { return &emptyChunkReader{} } return &singleChunkReader{ ctx: ctx, chunkLogger: chunkLogger, + generalLogger: generalLogger, slicePool: slicePool, cacheLimiter: cacheLimiter, sourceFactory: sourceFactory, @@ -118,21 +132,28 @@ func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFa } } -// Prefetch, and ignore any errors (just leave in not-prefetch-yet state, if there was an error) -// If we leave it in the not-prefetched state here, then when Read happens that will trigger another read attempt, -// and that one WILL return any error that happens -func (cr *singleChunkReader) TryBlockingPrefetch(fileReader io.ReaderAt) bool { - err := cr.blockingPrefetch(fileReader, false) - if err != nil { - cr.returnBuffer() // if there was an error, be sure to put us back into a valid "not-yet-prefetched" state - return false +// Use and un-use are temporary, for identifying the root cause of +// https://github.com/Azure/azure-storage-azcopy/issues/191 +// They may be removed after that. +// For now, they are used to wrap every Public method +func (cr *singleChunkReader) use() { + if atomic.SwapInt32(&cr.atomicUseIndicator, 1) != 0 { + panic("trying to use chunk reader when already in use") + } +} + +func (cr *singleChunkReader) unuse() { + if atomic.SwapInt32(&cr.atomicUseIndicator, 0) != 1 { + panic("ending use when chunk reader was not actually IN use") } - return true } func (cr *singleChunkReader) HasPrefetchedEntirelyZeros() bool { + cr.use() + defer cr.unuse() + if cr.buffer == nil { - return false // not prefetched (and, to simply error handling in teh caller, we don't call retryBlockingPrefetchIfNecessary here) + return false // not prefetched (and, to simply error handling in teh caller, we don't call retryBlockingPrefetchIfNecessary here) } for _, b := range cr.buffer { @@ -149,6 +170,13 @@ func (cr *singleChunkReader) HasPrefetchedEntirelyZeros() bool { // and (c) we would want to check whether it really did offer meaningful real-world performance gain, before introducing use of unsafe. } +func (cr *singleChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { + cr.use() + defer cr.unuse() + + return cr.blockingPrefetch(fileReader, isRetry) +} + // Prefetch the data in this chunk, using a file reader that is provided to us. // (Allowing the caller to provide the reader to us allows a sequential read approach, since caller can control the order sequentially (in the initial, non-retry, scenario) // We use io.ReaderAt, rather than io.Reader, just for maintainablity/ensuring correctness. (Since just using Reader requires the caller to @@ -169,10 +197,10 @@ func (cr *singleChunkReader) blockingPrefetch(fileReader io.ReaderAt, isRetry bo } // get buffer from pool - cr.buffer = cr.slicePool.RentSlice(uint32(cr.length)) + cr.buffer = cr.slicePool.RentSlice(uint32Checked(cr.length)) // read bytes into the buffer - cr.chunkLogger.LogChunkStatus(cr.chunkId, EWaitReason.Disk()) + cr.chunkLogger.LogChunkStatus(cr.chunkId, EWaitReason.DiskIO()) totalBytesRead, err := fileReader.ReadAt(cr.buffer, cr.chunkId.OffsetInFile) if err != nil && err != io.EOF { return err @@ -204,6 +232,8 @@ func (cr *singleChunkReader) retryBlockingPrefetchIfNecessary() error { // Seeks within this chunk // Seeking is used for retries, and also by some code to get length (by seeking to end) func (cr *singleChunkReader) Seek(offset int64, whence int) (int64, error) { + cr.use() + defer cr.unuse() newPosition := cr.positionInChunk @@ -229,6 +259,9 @@ func (cr *singleChunkReader) Seek(offset int64, whence int) (int64, error) { // Reads from within this chunk func (cr *singleChunkReader) Read(p []byte) (n int, err error) { + cr.use() + defer cr.unuse() + // This is a normal read, so free the prefetch buffer when hit EOF (i.e. end of this chunk). // We do so on the assumption that if we've read to the end we don't need the prefetched data any longer. // (If later, there's a retry that forces seek back to start and re-read, we'll automatically trigger a re-fetch at that time) @@ -250,6 +283,17 @@ func (cr *singleChunkReader) doRead(p []byte, freeBufferOnEof bool) (n int, err return 0, err } + // extra checks until we find root cause of https://github.com/Azure/azure-storage-azcopy/issues/191 + if cr.buffer == nil { + panic("unexpected nil buffer") + } + if cr.positionInChunk >= cr.length { + panic("unexpected EOF") + } + if cr.length != int64(len(cr.buffer)) { + panic("unexpected buffer length discrepancy") + } + // Copy the data across bytesCopied := copy(p, cr.buffer[cr.positionInChunk:]) cr.positionInChunk += int64(bytesCopied) @@ -276,6 +320,9 @@ func (cr *singleChunkReader) returnBuffer() { } func (cr *singleChunkReader) Length() int64 { + cr.use() + defer cr.unuse() + return cr.length } @@ -285,6 +332,26 @@ func (cr *singleChunkReader) Length() int64 { // Without this close, if something failed part way through, we would keep counting this object's bytes in cacheLimiter // "for ever", even after the object is gone. func (cr *singleChunkReader) Close() error { + // first, check and log early closes (before we do use(), since the situation we are trying + // to log is suspected to be one when use() will panic) + if cr.positionInChunk < cr.length { + // this is an "early close". Adjust logging verbosity depending on whether context is still active + var extraMessage string + if cr.ctx.Err() == nil { + b := &bytes.Buffer{} + b.Write(stack()) + extraMessage = "context active so logging full callstack, as follows: " + b.String() + } else { + extraMessage = "context cancelled so no callstack logged" + } + cr.generalLogger.Log(pipeline.LogInfo, "Early close of chunk in singleChunkReader: "+extraMessage) + } + + // after logging callstack, do normal use() + cr.use() + defer cr.unuse() + + // do the real work cr.returnBuffer() return nil } @@ -293,17 +360,55 @@ func (cr *singleChunkReader) Close() error { // (else we would have to re-read the start of the file later, and that breaks our rule to use sequential // reads as much as possible) func (cr *singleChunkReader) CaptureLeadingBytes() []byte { + cr.use() + // can't defer unuse here. See explict calls (plural) below + const mimeRecgonitionLen = 512 leadingBytes := make([]byte, mimeRecgonitionLen) n, err := cr.doRead(leadingBytes, false) // do NOT free bufferOnEOF. So that if its a very small file, and we hit the end, we won't needlessly discard the prefetched data if err != nil && err != io.EOF { + cr.unuse() return nil // we just can't sniff the mime type } if n < len(leadingBytes) { // truncate if we read less than expected (very small file, so err was EOF above) leadingBytes = leadingBytes[:n] } + // unuse before Seek, since Seek is public + cr.unuse() // MUST re-wind, so that the bytes we read will get transferred too! cr.Seek(0, io.SeekStart) return leadingBytes } + +func (cr *singleChunkReader) WriteBufferTo(h hash.Hash) { + cr.use() + defer cr.unuse() + + if cr.buffer == nil { + panic("invalid state. No prefetch buffer is present") + } + _, err := h.Write(cr.buffer) + if err != nil { + panic("documentation of hash.Hash.Write says it will never return an error") + } +} + +func stack() []byte { + buf := make([]byte, 2048) + for { + n := runtime.Stack(buf, false) + if n < len(buf) { + return buf[:n] + } + buf = make([]byte, 2*len(buf)) + } +} + +// while we never expect any out of range errors, due to chunk sizes fitting easily into uint32, here we make sure +func uint32Checked(i int64) uint32 { + if i > math.MaxUint32 { + panic("int64 out of range for cast to uint32") + } + return uint32(i) +} diff --git a/common/version.go b/common/version.go index 330dd5259..3686e9fec 100644 --- a/common/version.go +++ b/common/version.go @@ -1,4 +1,4 @@ package common -const AzcopyVersion = "10.0.7-Preview" +const AzcopyVersion = "10.0.8-Preview" const UserAgent = "AzCopy/" + AzcopyVersion diff --git a/common/zt_credCache_test.go b/common/zt_credCache_test.go index 8d776c721..4fe314b18 100644 --- a/common/zt_credCache_test.go +++ b/common/zt_credCache_test.go @@ -49,7 +49,12 @@ var fakeTokenInfo = OAuthTokenInfo{ } func (s *credCacheTestSuite) TestCredCacheSaveLoadDeleteHas(c *chk.C) { - credCache := NewCredCache(".") // "." state is reserved to be used in Linux and MacOS, and used as path to save token file in Windows. + credCache := NewCredCache(CredCacheOptions{ + DPAPIFilePath: "", + KeyName: "", + ServiceName: "", + AccountName: "", + }) // "." state is reserved to be used in Linux and MacOS, and used as path to save token file in Windows. defer func() { // Cleanup fake token diff --git a/common/zt_multiSliceBytePooler_test.go b/common/zt_multiSliceBytePooler_test.go new file mode 100644 index 000000000..817bc1667 --- /dev/null +++ b/common/zt_multiSliceBytePooler_test.go @@ -0,0 +1,66 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common + +import ( + chk "gopkg.in/check.v1" + "math" +) + +type multiSliceBytePoolerSuite struct{} + +var _ = chk.Suite(&multiSliceBytePoolerSuite{}) + +func (s *multiSliceBytePoolerSuite) TestMultiSliceSlotInfo(c *chk.C) { + eightMB := 8 * 1024 * 1024 + + cases := []struct { + size int + expectedSlotIndex int + expectedMaxCapInSlot int + }{ + {1, 0, 1}, + {2, 1, 2}, + {3, 2, 4}, + {4, 2, 4}, + {5, 3, 8}, + {8, 3, 8}, + {9, 4, 16}, + {eightMB - 1, 23, eightMB}, + {eightMB, 23, eightMB}, + {eightMB + 1, 24, eightMB * 2}, + {100 * 1024 * 1024, 27, 128 * 1024 * 1024}, + } + + for _, x := range cases { + logBase2 := math.Log2(float64(x.size)) + roundedLogBase2 := int(math.Round(logBase2 + 0.49999999999999)) // rounds up unless already exact(ish) + + // now lets see if the pooler is working as we expect + slotIndex, maxCap := getSlotInfo(uint32(x.size)) + + c.Assert(slotIndex, chk.Equals, roundedLogBase2) // this what, mathematically, we expect + c.Assert(slotIndex, chk.Equals, x.expectedSlotIndex) // this what our test case said (should be same) + + c.Assert(maxCap, chk.Equals, x.expectedMaxCapInSlot) + } + +} diff --git a/go.mod b/go.mod index 79de31b40..132abbbd0 100644 --- a/go.mod +++ b/go.mod @@ -2,19 +2,18 @@ module github.com/Azure/azure-storage-azcopy require ( github.com/Azure/azure-pipeline-go v0.1.8 - github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c + github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 github.com/Azure/azure-storage-file-go v0.0.0-20190108093629-d93e19c84c2a github.com/Azure/go-autorest v10.15.2+incompatible github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/danieljoos/wincred v1.0.1 - github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/inconshreveable/mousetrap v1.0.0 + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jiacfan/keychain v0.0.0-20180920053336-f2c902a3d807 github.com/jiacfan/keyctl v0.0.0-20160328205232-988d05162bc5 - github.com/kr/pretty v0.1.0 - github.com/kr/text v0.1.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.2 + github.com/stretchr/testify v1.3.0 // indirect golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 ) diff --git a/go.sum b/go.sum index 7f80d2b31..156ae7236 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,9 @@ -github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876 h1:3c3mGlhASTJh6H6Ba9EHv2FDSmEUyJuJHR6UD7b+YuE= -github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/xg++KS20= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-storage-blob-go v0.0.0-20180727221336-197d1c0aea1b h1:7cOe9XtL/0qFd/jb0whfm5NoRmhEcVU3bZ65zUZUz54= -github.com/Azure/azure-storage-blob-go v0.0.0-20180727221336-197d1c0aea1b/go.mod h1:x2mtS6O3mnMEZOJp7d7oldh8IvatBrMfReiyQ+cKgKY= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc h1:BElWmFfsryQD72OcovStKpkIcd4e9ozSkdsTNQDSHGk= -github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 h1:QjGHsWFbJyl312t0BtgkmZy2TTYA++FF0UakGbr3ZhQ= +github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-file-go v0.0.0-20180929015327-d6c64f9676be h1:2TBD/QJYxkQf2jAHk/radyLgEfUpV8eqvRPvnjJt9EA= github.com/Azure/azure-storage-file-go v0.0.0-20180929015327-d6c64f9676be/go.mod h1:N5mXnKL8ZzcrxcNfqrcfWhiaCPAGagfTxH0/IwPN/LI= github.com/Azure/azure-storage-file-go v0.0.0-20190108093629-d93e19c84c2a h1:5OfEqciJHSMkxAWgJP1b3JTmzWNRlq9L9IgOxPNlBOM= @@ -18,6 +14,8 @@ github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNP github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/danieljoos/wincred v1.0.1 h1:fcRTaj17zzROVqni2FiToKUVg3MmJ4NtMSGCySPIr/g= github.com/danieljoos/wincred v1.0.1/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -31,10 +29,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190107173414-20be8e55dc7b h1:9Gu1sMPgKHo+qCbPa2jN5A54ro2gY99BWF7nHOBNVME= diff --git a/main.go b/main.go index 0a162b385..72846dfa9 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,8 @@ import ( "log" "os" "runtime" + "runtime/debug" + "time" ) // get the lifecycle manager to print messages @@ -51,7 +53,21 @@ func main() { log.Fatalf("initialization failed: %v", err) } - ste.MainSTE(common.ComputeConcurrencyValue(runtime.NumCPU()), 2400, azcopyAppPathFolder, azcopyLogPathFolder) + configureGC() + + err = ste.MainSTE(common.ComputeConcurrencyValue(runtime.NumCPU()), 2400, azcopyAppPathFolder, azcopyLogPathFolder) + common.PanicIfErr(err) + cmd.Execute(azcopyAppPathFolder, azcopyLogPathFolder) - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) +} + +// Golang's default behaviour is to GC when new objects = (100% of) total of objects surviving previous GC. +// But our "survivors" add up to many GB, so its hard for users to be confident that we don't have +// a memory leak (since with that default setting new GCs are very rare in our case). So configure them to be more frequent. +func configureGC() { + go func() { + time.Sleep(20 * time.Second) // wait a little, so that our initial pool of buffers can get allocated without heaps of (unnecessary) GC activity + debug.SetGCPercent(20) // activate more aggressive/frequent GC than the default + }() } diff --git a/ste/ErrorExt.go b/ste/ErrorExt.go index b3fcc567d..d84e7edfe 100644 --- a/ste/ErrorExt.go +++ b/ste/ErrorExt.go @@ -4,12 +4,14 @@ import ( "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-storage-file-go/azfile" + "net/http" ) type ErrorEx struct { error } +// TODO: consider rolling MSRequestID into this, so that all places that use this can pick up, and log, the request ID too func (errex ErrorEx) ErrorCodeAndString() (int, string) { switch e := interface{}(errex.error).(type) { case azblob.StorageError: @@ -22,3 +24,19 @@ func (errex ErrorEx) ErrorCodeAndString() (int, string) { return 0, errex.Error() } } + +type hasResponse interface { + Response() *http.Response +} + +// MSRequestID gets the request ID guid associated with the failed request. +// Returns "" if there isn't one (either no request, or there is a request but it doesn't have the header) +func (errex ErrorEx) MSRequestID() string { + if respErr, ok := errex.error.(hasResponse); ok { + r := respErr.Response() + if r != nil { + return r.Header.Get("X-Ms-Request-Id") + } + } + return "" +} diff --git a/ste/JobPartPlan.go b/ste/JobPartPlan.go index e28857d7a..7d343b7bb 100644 --- a/ste/JobPartPlan.go +++ b/ste/JobPartPlan.go @@ -14,7 +14,7 @@ import ( // dataSchemaVersion defines the data schema version of JobPart order files supported by // current version of azcopy // To be Incremented every time when we release azcopy with changed dataSchema -const DataSchemaVersion common.Version = 1 +const DataSchemaVersion common.Version = 3 const ( ContentTypeMaxBytes = 256 // If > 65536, then jobPartPlanBlobData's ContentTypeLength's type field must change @@ -39,20 +39,24 @@ func (mmf *JobPartPlanMMF) Unmap() { (*common.MMF)(mmf).Unmap() } // JobPartPlanHeader represents the header of Job Part's memory-mapped file type JobPartPlanHeader struct { // Once set, the following fields are constants; they should never be modified - Version common.Version // The version of data schema format of header; see the dataSchemaVersion constant - StartTime int64 // The start time of this part - JobID common.JobID // Job Part's JobID - PartNum common.PartNumber // Job Part's part number (0+) - IsFinalPart bool // True if this is the Job's last part; else false - ForceWrite bool // True if the existing blobs needs to be overwritten. - Priority common.JobPriority // The Job Part's priority - TTLAfterCompletion uint32 // Time to live after completion is used to persists the file on disk of specified time after the completion of JobPartOrder - FromTo common.FromTo // The location of the transfer's source & destination - CommandStringLength uint32 - NumTransfers uint32 // The number of transfers in the Job part - LogLevel common.LogLevel // This Job Part's minimal log level - DstBlobData JobPartPlanDstBlob // Additional data for blob destinations - DstLocalData JobPartPlanDstLocal // Additional data for local destinations + Version common.Version // The version of data schema format of header; see the dataSchemaVersion constant + StartTime int64 // The start time of this part + JobID common.JobID // Job Part's JobID + PartNum common.PartNumber // Job Part's part number (0+) + SourceRootLength uint16 // The length of the source root path + SourceRoot [1000]byte // The root directory of the source + DestinationRootLength uint16 // The length of the destination root path + DestinationRoot [1000]byte // The root directory of the destination + IsFinalPart bool // True if this is the Job's last part; else false + ForceWrite bool // True if the existing blobs needs to be overwritten. + Priority common.JobPriority // The Job Part's priority + TTLAfterCompletion uint32 // Time to live after completion is used to persists the file on disk of specified time after the completion of JobPartOrder + FromTo common.FromTo // The location of the transfer's source & destination + CommandStringLength uint32 + NumTransfers uint32 // The number of transfers in the Job part + LogLevel common.LogLevel // This Job Part's minimal log level + DstBlobData JobPartPlanDstBlob // Additional data for blob destinations + DstLocalData JobPartPlanDstLocal // Additional data for local destinations // Any fields below this comment are NOT constants; they may change over as the job part is processed. // Care must be taken to read/write to these fields in a thread-safe way! @@ -97,6 +101,9 @@ func (jpph *JobPartPlanHeader) CommandString() string { // TransferSrcDstDetail returns the source and destination string for a transfer at given transferIndex in JobPartOrder func (jpph *JobPartPlanHeader) TransferSrcDstStrings(transferIndex uint32) (source, destination string) { + srcRoot := string(jpph.SourceRoot[:jpph.SourceRootLength]) + dstRoot := string(jpph.DestinationRoot[:jpph.DestinationRootLength]) + jppt := jpph.Transfer(transferIndex) srcSlice := []byte{} @@ -111,7 +118,7 @@ func (jpph *JobPartPlanHeader) TransferSrcDstStrings(transferIndex uint32) (sour sh.Len = int(jppt.DstLength) sh.Cap = sh.Len - return string(srcSlice), string(dstSlice) + return srcRoot + string(srcSlice), dstRoot + string(dstSlice) } func (jpph *JobPartPlanHeader) getString(offset int64, length int16) string { @@ -212,6 +219,9 @@ type JobPartPlanDstLocal struct { // Specifies whether the timestamp of destination file has to be set to the modified time of source file PreserveLastModifiedTime bool + + // says how MD5 verification failures should be actioned + MD5VerificationOption common.HashValidationOption } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/ste/JobPartPlanFileName.go b/ste/JobPartPlanFileName.go index ff0ecb218..0468f807e 100644 --- a/ste/JobPartPlanFileName.go +++ b/ste/JobPartPlanFileName.go @@ -70,14 +70,20 @@ func (jpfn JobPartPlanFileName) Map() *JobPartPlanMMF { // createJobPartPlanFile creates the memory map JobPartPlanHeader using the given JobPartOrder and JobPartPlanBlobData func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { // Validate that the passed-in strings can fit in their respective fields + if len(order.SourceRoot) > len(JobPartPlanHeader{}.SourceRoot) { + panic(fmt.Errorf("source root string is too large: %q", order.SourceRoot)) + } + if len(order.DestinationRoot) > len(JobPartPlanHeader{}.DestinationRoot) { + panic(fmt.Errorf("destination root string is too large: %q", order.DestinationRoot)) + } if len(order.BlobAttributes.ContentType) > len(JobPartPlanDstBlob{}.ContentType) { - panic(fmt.Errorf("content type string it too large: %q", order.BlobAttributes.ContentType)) + panic(fmt.Errorf("content type string is too large: %q", order.BlobAttributes.ContentType)) } if len(order.BlobAttributes.ContentEncoding) > len(JobPartPlanDstBlob{}.ContentEncoding) { - panic(fmt.Errorf("content encoding string it too large: %q", order.BlobAttributes.ContentEncoding)) + panic(fmt.Errorf("content encoding string is too large: %q", order.BlobAttributes.ContentEncoding)) } if len(order.BlobAttributes.Metadata) > len(JobPartPlanDstBlob{}.Metadata) { - panic(fmt.Errorf("metadata string it too large: %q", order.BlobAttributes.Metadata)) + panic(fmt.Errorf("metadata string is too large: %q", order.BlobAttributes.Metadata)) } // This nested function writes a structure value to an io.Writer & returns the number of bytes written @@ -131,18 +137,20 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { //} // Initialize the Job Part's Plan header jpph := JobPartPlanHeader{ - Version: DataSchemaVersion, - StartTime: time.Now().UnixNano(), - JobID: order.JobID, - PartNum: order.PartNum, - IsFinalPart: order.IsFinalPart, - ForceWrite: order.ForceWrite, - Priority: order.Priority, - TTLAfterCompletion: uint32(time.Time{}.Nanosecond()), - FromTo: order.FromTo, - CommandStringLength: uint32(len(order.CommandString)), - NumTransfers: uint32(len(order.Transfers)), - LogLevel: order.LogLevel, + Version: DataSchemaVersion, + StartTime: time.Now().UnixNano(), + JobID: order.JobID, + PartNum: order.PartNum, + SourceRootLength: uint16(len(order.SourceRoot)), + DestinationRootLength: uint16(len(order.DestinationRoot)), + IsFinalPart: order.IsFinalPart, + ForceWrite: order.ForceWrite, + Priority: order.Priority, + TTLAfterCompletion: uint32(time.Time{}.Nanosecond()), + FromTo: order.FromTo, + CommandStringLength: uint32(len(order.CommandString)), + NumTransfers: uint32(len(order.Transfers)), + LogLevel: order.LogLevel, DstBlobData: JobPartPlanDstBlob{ BlobType: order.BlobAttributes.BlobType, NoGuessMimeType: order.BlobAttributes.NoGuessMimeType, @@ -155,11 +163,14 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { }, DstLocalData: JobPartPlanDstLocal{ PreserveLastModifiedTime: order.BlobAttributes.PreserveLastModifiedTime, + MD5VerificationOption: order.BlobAttributes.MD5ValidationOption, }, atomicJobStatus: common.EJobStatus.InProgress(), // We default to InProgress } // Copy any strings into their respective fields + copy(jpph.SourceRoot[:], order.SourceRoot) + copy(jpph.DestinationRoot[:], order.DestinationRoot) copy(jpph.DstBlobData.ContentType[:], order.BlobAttributes.ContentType) copy(jpph.DstBlobData.ContentEncoding[:], order.BlobAttributes.ContentEncoding) copy(jpph.DstBlobData.Metadata[:], order.BlobAttributes.Metadata) @@ -225,7 +236,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { jppt.SrcCacheControlLength + jppt.SrcContentMD5Length + jppt.SrcMetadataLength + jppt.SrcBlobTypeLength) } - // All the transfers were written; now write each each transfer's src/dst strings + // All the transfers were written; now write each transfer's src/dst strings for t := range order.Transfers { // Sanity check: Verify that we are were we think we are and that no bug has occurred if eof != srcDstStringsOffset[t] { @@ -241,7 +252,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { common.PanicIfErr(err) eof += int64(bytesWritten) - // For S2S copy, write the src properties + // For S2S copy (and, in the case of Content-MD5, always), write the src properties if len(order.Transfers[t].ContentType) != 0 { bytesWritten, err = file.WriteString(order.Transfers[t].ContentType) common.PanicIfErr(err) @@ -267,7 +278,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { common.PanicIfErr(err) eof += int64(bytesWritten) } - if order.Transfers[t].ContentMD5 != nil { + if order.Transfers[t].ContentMD5 != nil { // if non-nil but 0 len, will simply not be read by the consumer (since length is zero) bytesWritten, err = file.WriteString(string(order.Transfers[t].ContentMD5)) common.PanicIfErr(err) eof += int64(bytesWritten) diff --git a/ste/JobsAdmin.go b/ste/JobsAdmin.go index 9ddbdb2f4..06c8ac10a 100644 --- a/ste/JobsAdmin.go +++ b/ste/JobsAdmin.go @@ -130,10 +130,12 @@ func initJobsAdmin(appCtx context.Context, concurrentConnections int, targetRate // TODO: make ram usage configurable, with the following as just the default // Decide on a max amount of RAM we are willing to use. This functions as a cap, and prevents excessive usage. // There's no measure of physical RAM in the STD library, so we guestimate conservatively, based on CPU count (logical, not phyiscal CPUs) - const gbToUsePerCpu = 0.6 // should be enough to support the amount of traffic 1 CPU can drive, and also less than the typical installed RAM-per-CPU + // Note that, as at Feb 2019, the multiSizeSlicePooler uses additional RAM, over this level, since it includes the cache of + // currently-unnused, re-useable slices, that is not tracked by cacheLimiter. + const gbToUsePerCpu = 0.5 // should be enough to support the amount of traffic 1 CPU can drive, and also less than the typical installed RAM-per-CPU gbToUse := float32(runtime.NumCPU()) * gbToUsePerCpu - if gbToUse > 8 { - gbToUse = 8 // cap it. We don't need more than this. Even 6 is enough at 10 Gbps with standard chunk sizes, but allow a little extra here to help if larger blob block sizes are selected by user + if gbToUse > 10 { + gbToUse = 10 // cap it. Even 6 is enough at 10 Gbps with standard chunk sizes, but allow a little extra here to help if larger blob block sizes are selected by user } maxRamBytesToUse := int64(gbToUse * 1024 * 1024 * 1024) @@ -165,6 +167,9 @@ func initJobsAdmin(appCtx context.Context, concurrentConnections int, targetRate JobsAdmin = ja + // Spin up slice pool pruner + go ja.slicePoolPruneLoop() + // One routine constantly monitors the partsChannel. It takes the JobPartManager from // the Channel and schedules the transfers of that JobPart. go ja.scheduleJobParts() @@ -212,7 +217,7 @@ func (ja *jobsAdmin) chunkProcessor(workerID int) { // We check for suicides first to shrink goroutine pool // Then, we check chunks: normal & low priority select { - case <-ja.xferChannels.suicideCh: // note: as at Dec 2018, this channel is not (yet) used + case <-ja.xferChannels.suicideCh: // note: as at Dec 2018, this channel is not (yet) used return default: select { @@ -224,10 +229,10 @@ func (ja *jobsAdmin) chunkProcessor(workerID int) { chunkFunc(workerID) default: time.Sleep(100 * time.Millisecond) // Sleep before looping around - // TODO: Question: In order to safely support high goroutine counts, - // do we need to review sleep duration, or find an approach that does not require waking every x milliseconds - // For now, duration has been increased substantially from the previous 1 ms, to reduce cost of - // the wake-ups. + // TODO: Question: In order to safely support high goroutine counts, + // do we need to review sleep duration, or find an approach that does not require waking every x milliseconds + // For now, duration has been increased substantially from the previous 1 ms, to reduce cost of + // the wake-ups. } } } @@ -282,7 +287,7 @@ type jobsAdmin struct { xferChannels XferChannels appCtx context.Context pacer *pacer - slicePool common.ByteSlicePooler + slicePool common.ByteSlicePooler cacheLimiter common.CacheLimiter } @@ -478,6 +483,23 @@ func (ja *jobsAdmin) Log(level pipeline.LogLevel, msg string) { ja.logger.Log(le func (ja *jobsAdmin) Panic(err error) { ja.logger.Panic(err) } func (ja *jobsAdmin) CloseLog() { ja.logger.CloseLog() } +func (ja *jobsAdmin) slicePoolPruneLoop() { + // if something in the pool has been unused for this long, we probably don't need it + const pruneInterval = 5 * time.Second + + ticker := time.NewTicker(pruneInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + ja.slicePool.Prune() + case <-ja.appCtx.Done(): + break + } + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // The jobIDToJobMgr maps each JobID to its JobMgr diff --git a/ste/downloader-azureFiles.go b/ste/downloader-azureFiles.go index 3580b59ab..04522043b 100644 --- a/ste/downloader-azureFiles.go +++ b/ste/downloader-azureFiles.go @@ -21,6 +21,7 @@ package ste import ( + "errors" "net/url" "github.com/Azure/azure-pipeline-go/pipeline" @@ -34,7 +35,7 @@ func newAzureFilesDownloader() downloader { return &azureFilesDownloader{} } -// Returns a chunk-func for file downloads +// GenerateDownloadFunc returns a chunk-func for file downloads func (bd *azureFilesDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipeline pipeline.Pipeline, destWriter common.ChunkedFileWriter, id common.ChunkID, length int64, pacer *pacer) chunkFunc { return createDownloadChunkFunc(jptm, id, func() { @@ -53,17 +54,22 @@ func (bd *azureFilesDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, s return } + // Verify that the file has not been changed via a client side LMT check + getLocation := get.LastModified().Location() + if !get.LastModified().Equal(jptm.LastModifiedTime().In(getLocation)) { + jptm.FailActiveDownload("Azure File modified during transfer", + errors.New("Azure File modified during transfer")) + } + // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) - retryReader := get.Body(azfile.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) + retryReader := get.Body(azfile.RetryReaderOptions{ + MaxRetryRequests: MaxRetryPerDownloadBody, + NotifyFailedRead: common.NewReadLogFunc(jptm, u), + }) defer retryReader.Close() - retryForcer := func() { - // TODO: implement this, or implement GetBodyWithForceableRetry above - // for now, this "retry forcer" does nothing - //fmt.Printf("\nForcing retry\n") - } - err = destWriter.EnqueueChunk(jptm.Context(), retryForcer, id, length, newLiteResponseBodyPacer(retryReader, pacer)) + err = destWriter.EnqueueChunk(jptm.Context(), id, length, newLiteResponseBodyPacer(retryReader, pacer), true) if err != nil { jptm.FailActiveDownload("Enqueuing chunk", err) return diff --git a/ste/downloader-blob.go b/ste/downloader-blob.go index 6e860b71f..bebe0f404 100644 --- a/ste/downloader-blob.go +++ b/ste/downloader-blob.go @@ -45,7 +45,9 @@ func (bd *blobDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipe // wait until we get the headers back... but we have not yet read its whole body. // The Download method encapsulates any retries that may be necessary to get to the point of receiving response headers. jptm.LogChunkStatus(id, common.EWaitReason.HeaderResponse()) - get, err := srcBlobURL.Download(jptm.Context(), id.OffsetInFile, length, azblob.BlobAccessConditions{}, false) + get, err := srcBlobURL.Download(jptm.Context(), id.OffsetInFile, length, + azblob.BlobAccessConditions{ModifiedAccessConditions: + azblob.ModifiedAccessConditions{IfUnmodifiedSince:jptm.LastModifiedTime()}}, false) if err != nil { jptm.FailActiveDownload("Downloading response body", err) // cancel entire transfer because this chunk has failed return @@ -54,13 +56,12 @@ func (bd *blobDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipe // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) - //TODO: retryReader, retryForcer := get.BodyWithForceableRetry(azblob.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) - retryReader := get.Body(azblob.RetryReaderOptions{MaxRetryRequests: destWriter.MaxRetryPerDownloadBody()}) - retryForcer := func() {} - // TODO: replace the above with real retry forcer - + retryReader := get.Body(azblob.RetryReaderOptions{ + MaxRetryRequests: destWriter.MaxRetryPerDownloadBody(), + NotifyFailedRead: common.NewReadLogFunc(jptm, u), + }) defer retryReader.Close() - err = destWriter.EnqueueChunk(jptm.Context(), retryForcer, id, length, newLiteResponseBodyPacer(retryReader, pacer)) + err = destWriter.EnqueueChunk(jptm.Context(), id, length, newLiteResponseBodyPacer(retryReader, pacer), true) if err != nil { jptm.FailActiveDownload("Enqueuing chunk", err) return diff --git a/ste/downloader-blobFS.go b/ste/downloader-blobFS.go index e49883be9..2b7bde909 100644 --- a/ste/downloader-blobFS.go +++ b/ste/downloader-blobFS.go @@ -21,10 +21,12 @@ package ste import ( + "errors" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-azcopy/common" "net/url" + "time" ) type blobFSDownloader struct{} @@ -52,17 +54,26 @@ func (bd *blobFSDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPi return } + // parse the remote lmt, there shouldn't be any error, unless the service returned a new format + remoteLastModified, err := time.Parse(time.RFC1123, get.LastModified()) + common.PanicIfErr(err) + remoteLmtLocation := remoteLastModified.Location() + + // Verify that the file has not been changed via a client side LMT check + if !remoteLastModified.Equal(jptm.LastModifiedTime().In(remoteLmtLocation)) { + jptm.FailActiveDownload("BFS File modified during transfer", + errors.New("BFS File modified during transfer")) + } + // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) - retryReader := get.Body(azbfs.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) + retryReader := get.Body(azbfs.RetryReaderOptions{ + MaxRetryRequests: MaxRetryPerDownloadBody, + NotifyFailedRead: common.NewReadLogFunc(jptm, u), + }) defer retryReader.Close() - retryForcer := func() { - // TODO: implement this, or implement GetBodyWithForceableRetry above - // for now, this "retry forcer" does nothing - //fmt.Printf("\nForcing retry\n") - } - err = destWriter.EnqueueChunk(jptm.Context(), retryForcer, id, length, newLiteResponseBodyPacer(retryReader, pacer)) + err = destWriter.EnqueueChunk(jptm.Context(), id, length, newLiteResponseBodyPacer(retryReader, pacer), true) if err != nil { jptm.FailActiveDownload("Enqueuing chunk", err) return diff --git a/ste/init.go b/ste/init.go index ddb195ae9..5ab761cbd 100644 --- a/ste/init.go +++ b/ste/init.go @@ -458,6 +458,7 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { // Get the number of active go routines performing the transfer or executing the chunk Func // TODO: added for debugging purpose. remove later js.ActiveConnections = jm.ActiveConnections() + js.PerfStrings, js.IsDiskConstrained = jm.GetPerfInfo() // If the status is cancelled, then no need to check for completerJobOrdered // since user must have provided the consent to cancel an incompleteJob if that @@ -472,6 +473,12 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { if (js.CompleteJobOrdered) && (part0PlanStatus == common.EJobStatus.Completed()) { js.JobStatus = part0PlanStatus } + + if js.JobStatus == common.EJobStatus.Completed() { + js.JobStatus = js.JobStatus.EnhanceJobStatusInfo(js.TransfersSkipped > 0, js.TransfersFailed > 0, + js.TransfersCompleted > 0) + } + return js } @@ -487,6 +494,9 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { * DeleteTransfersCompleted - number of delete transfers failed in the job. * FailedTransfers - list of transfer that failed. */ +// TODO determine if this should be removed, since we currently perform the deletions in the enumeration engine +// TODO if deletions are also done in the backend, then we should keep this & improve it potentially +// TODO deletions and copies can currently be placed in different job parts func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { // getJobPartMapFromJobPartInfoMap gives the map of partNo to JobPartPlanInfo Pointer for a given JobId jm, found := JobsAdmin.JobMgr(jobID) @@ -535,6 +545,8 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if fromTo == common.EFromTo.LocalBlob() || fromTo == common.EFromTo.BlobLocal() { js.CopyTransfersCompleted++ + js.TotalBytesTransferred += uint64(jppt.SourceSize) + js.TotalBytesEnumerated += uint64(jppt.SourceSize) } if fromTo == common.EFromTo.BlobTrash() { js.DeleteTransfersCompleted++ @@ -545,6 +557,7 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if fromTo == common.EFromTo.LocalBlob() || fromTo == common.EFromTo.BlobLocal() { js.CopyTransfersFailed++ + js.TotalBytesEnumerated += uint64(jppt.SourceSize) } if fromTo == common.EFromTo.BlobTrash() { js.DeleteTransfersFailed++ @@ -577,6 +590,7 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { // Get the number of active go routines performing the transfer or executing the chunk Func // TODO: added for debugging purpose. remove later js.ActiveConnections = jm.ActiveConnections() + js.PerfStrings, js.IsDiskConstrained = jm.GetPerfInfo() // If the status is cancelled, then no need to check for completerJobOrdered // since user must have provided the consent to cancel an incompleteJob if that @@ -591,6 +605,13 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if (js.CompleteJobOrdered) && (part0PlanStatus == common.EJobStatus.Completed()) { js.JobStatus = part0PlanStatus } + + if js.JobStatus == common.EJobStatus.Completed() { + js.JobStatus = js.JobStatus.EnhanceJobStatusInfo(false, + js.CopyTransfersFailed+js.DeleteTransfersFailed > 0, + js.CopyTransfersCompleted+js.DeleteTransfersCompleted > 0) + } + return js } diff --git a/ste/md5Comparer.go b/ste/md5Comparer.go new file mode 100644 index 000000000..fab8a1ee5 --- /dev/null +++ b/ste/md5Comparer.go @@ -0,0 +1,103 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package ste + +import ( + "bytes" + "errors" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" +) + +type transferSpecificLogger interface { + LogAtLevelForCurrentTransfer(level pipeline.LogLevel, msg string) +} + +type md5Comparer struct { + expected []byte + actualAsSaved []byte + validationOption common.HashValidationOption + logger transferSpecificLogger +} + +// TODO: let's add an aka.ms link to the message, that gives more info +var errMd5Mismatch = errors.New("the MD5 hash of the data, as we received it, did not match the expected value, as found in the Blob/File Service. " + + "This means that either there is a data integrity error OR another tool has failed to keep the stored hash up to date") + +// TODO: let's add an aka.ms link to the message, that gives more info +const noMD5Stored = "no MD5 was stored in the Blob/File service against this file. So the downloaded data cannot be MD5-validated." + +var errExpectedMd5Missing = errors.New(noMD5Stored + " This application is currently configured to treat missing MD5 hashes as errors") + +var errActualMd5NotComputed = errors.New("no MDB was computed within this application. This indicates a logic error in this application") + +// Check compares the two MD5s, and returns any error if applicable +// Any informational logging will be done within Check, so all the caller needs to do +// is respond to non-nil errors +func (c *md5Comparer) Check() error { + + if c.validationOption == common.EHashValidationOption.NoCheck() { + return nil + } + + if c.actualAsSaved == nil || len(c.actualAsSaved) == 0 { + return errActualMd5NotComputed // Should never happen, so there's no way to opt out of this error being returned if it DOES happen + } + + // missing + if c.expected == nil || len(c.expected) == 0 { + switch c.validationOption { + case common.EHashValidationOption.FailIfDifferentOrMissing(): + return errExpectedMd5Missing + case common.EHashValidationOption.FailIfDifferent(), + common.EHashValidationOption.LogOnly(): + c.logAsMissing() + return nil + default: + panic("unexpected hash validation type") + } + } + + // different + match := bytes.Equal(c.expected, c.actualAsSaved) + if !match { + switch c.validationOption { + case common.EHashValidationOption.FailIfDifferentOrMissing(), + common.EHashValidationOption.FailIfDifferent(): + return errMd5Mismatch + case common.EHashValidationOption.LogOnly(): + c.logAsDifferent() + return nil + default: + panic("unexpected hash validation type") + } + } + + return nil +} + +func (c *md5Comparer) logAsMissing() { + c.logger.LogAtLevelForCurrentTransfer(pipeline.LogWarning, noMD5Stored) +} + +func (c *md5Comparer) logAsDifferent() { + c.logger.LogAtLevelForCurrentTransfer(pipeline.LogWarning, errMd5Mismatch.Error()) +} diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index f099090a3..f6a799fba 100644 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -23,6 +23,7 @@ package ste import ( "context" "fmt" + "strings" "sync" "sync/atomic" @@ -65,6 +66,7 @@ type IJobMgr interface { ReleaseAConnection() // TODO: added for debugging purpose. remove later ActiveConnections() int64 + GetPerfInfo() (displayStrings []string, isDiskConstrained bool) //Close() getInMemoryTransitJobState() InMemoryTransitJobState // get in memory transit job state saved in this job. setInMemoryTransitJobState(state InMemoryTransitJobState) // set in memory transit job state saved in this job. @@ -77,10 +79,10 @@ type IJobMgr interface { func newJobMgr(appLogger common.ILogger, jobID common.JobID, appCtx context.Context, level common.LogLevel, commandString string, logFileFolder string) IJobMgr { // atomicAllTransfersScheduled is set to 1 since this api is also called when new job part is ordered. - enableChunkLog := level.ToPipelineLogLevel() == pipeline.LogDebug + enableChunkLogOutput := level.ToPipelineLogLevel() == pipeline.LogDebug jm := jobMgr{jobID: jobID, jobPartMgrs: newJobPartToJobPartMgr(), include: map[string]int{}, exclude: map[string]int{}, - logger: common.NewJobLogger(jobID, level, appLogger, logFileFolder), - chunkStatusLogger: common.NewChunkStatusLogger(jobID, logFileFolder, enableChunkLog), + logger: common.NewJobLogger(jobID, level, appLogger, logFileFolder), + chunkStatusLogger: common.NewChunkStatusLogger(jobID, logFileFolder, enableChunkLogOutput), /*Other fields remain zero-value until this job is scheduled */} jm.reset(appCtx, commandString) return &jm @@ -103,11 +105,11 @@ func (jm *jobMgr) reset(appCtx context.Context, commandString string) IJobMgr { // jobMgr represents the runtime information for a Job type jobMgr struct { - logger common.ILoggerResetable + logger common.ILoggerResetable chunkStatusLogger common.ChunkStatusLoggerCloser - jobID common.JobID // The Job's unique ID - ctx context.Context - cancel context.CancelFunc + jobID common.JobID // The Job's unique ID + ctx context.Context + cancel context.CancelFunc jobPartMgrs jobPartToJobPartMgr // The map of part #s to JobPartMgrs // partsDone keep the count of completed part of the Job. @@ -128,6 +130,8 @@ type jobMgr struct { // atomicCurrentConcurrentConnections defines the number of active goroutines performing the transfer / executing the chunk func // TODO: added for debugging purpose. remove later atomicCurrentConcurrentConnections int64 + atomicIsUploadIndicator int32 + atomicIsDownloadIndicator int32 } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -165,16 +169,57 @@ func (jm *jobMgr) ActiveConnections() int64 { return atomic.LoadInt64(&jm.atomicCurrentConcurrentConnections) } +// GetPerfStrings returns strings that may be logged for performance diagnostic purposes +// The number and content of strings may change as we enhance our perf diagnostics +func (jm *jobMgr) GetPerfInfo() (displayStrings []string, isDiskConstrained bool) { + + // get data appropriate to our current transfer direction + isUpload := atomic.LoadInt32(&jm.atomicIsUploadIndicator) == 1 + isDownload := atomic.LoadInt32(&jm.atomicIsDownloadIndicator) == 1 + chunkStateCounts := jm.chunkStatusLogger.GetCounts(isDownload) + + // convert the counts to simple strings for consumption by callers + const format = "%c: %2d" + result := make([]string, len(chunkStateCounts)+1) + total := int64(0) + for i, c := range chunkStateCounts { + result[i] = fmt.Sprintf(format, c.WaitReason.Name[0], c.Count) + total += c.Count + } + result[len(result)-1] = fmt.Sprintf(format, 'T', total) + + diskCon := jm.chunkStatusLogger.IsDiskConstrained(isUpload, isDownload) + + // logging from here is a bit of a hack + // TODO: can we find a better way to get this info into the log? The caller is at app level, + // not job level, so can't log it directly AFAICT. + jm.logPerfInfo(result, diskCon) + + return result, diskCon +} + +func (jm *jobMgr) logPerfInfo(displayStrings []string, isDiskConstrained bool) { + var diskString string + if isDiskConstrained { + diskString = "disk MAY BE limiting throughput" + } else { + diskString = "disk IS NOT limiting throughput" + } + msg := fmt.Sprintf("PERF: disk %s. States: %s", diskString, strings.Join(displayStrings, ", ")) + jm.Log(pipeline.LogInfo, msg) +} + // initializeJobPartPlanInfo func initializes the JobPartPlanInfo handler for given JobPartOrder func (jm *jobMgr) AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, sourceSAS string, destinationSAS string, scheduleTransfers bool) IJobPartMgr { jpm := &jobPartMgr{jobMgr: jm, filename: planFile, sourceSAS: sourceSAS, destinationSAS: destinationSAS, pacer: JobsAdmin.(*jobsAdmin).pacer, - slicePool: JobsAdmin.(*jobsAdmin).slicePool, + slicePool: JobsAdmin.(*jobsAdmin).slicePool, cacheLimiter: JobsAdmin.(*jobsAdmin).cacheLimiter} jpm.planMMF = jpm.filename.Map() jm.jobPartMgrs.Set(partNum, jpm) jm.finalPartOrdered = jpm.planMMF.Plan().IsFinalPart + jm.setDirection(jpm.Plan().FromTo) if scheduleTransfers { // If the schedule transfer is set to true // Instead of the scheduling the Transfer for given JobPart @@ -186,6 +231,21 @@ func (jm *jobMgr) AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, s return jpm } +// Remembers which direction we are running in (upload, download or neither (for service to service)) +// It actually remembers the direction that our most recently-added job PART is running in, +// because that's where the fromTo information can be found, +// but we assume taht all the job parts are running in the same direction +func (jm *jobMgr) setDirection(fromTo common.FromTo) { + fromIsLocal := fromTo.From() == common.ELocation.Local() + toIsLocal := fromTo.To() == common.ELocation.Local() + + isUpload := fromIsLocal && !toIsLocal + isDownload := !fromIsLocal && toIsLocal + + atomic.StoreInt32(&jm.atomicIsUploadIndicator, common.Iffint32(isUpload, 1, 0)) + atomic.StoreInt32(&jm.atomicIsDownloadIndicator, common.Iffint32(isDownload, 1, 0)) +} + // SetIncludeExclude sets the include / exclude list of transfers // supplied with resume command to include or exclude mentioned transfers func (jm *jobMgr) SetIncludeExclude(include, exclude map[string]int) { @@ -280,16 +340,15 @@ func (jm *jobMgr) PipelineLogInfo() pipeline.LogOptions { } } func (jm *jobMgr) Panic(err error) { jm.logger.Panic(err) } -func (jm *jobMgr) CloseLog(){ +func (jm *jobMgr) CloseLog() { jm.logger.CloseLog() jm.chunkStatusLogger.CloseLog() } -func (jm *jobMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason){ +func (jm *jobMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason) { jm.chunkStatusLogger.LogChunkStatus(id, reason) } - // PartsDone returns the number of the Job's parts that are either completed or failed //func (jm *jobMgr) PartsDone() uint32 { return atomic.LoadUint32(&jm.partsDone) } diff --git a/ste/mgr-JobPartMgr.go b/ste/mgr-JobPartMgr.go index 62e22076a..ea219b8a3 100644 --- a/ste/mgr-JobPartMgr.go +++ b/ste/mgr-JobPartMgr.go @@ -421,16 +421,16 @@ func (jpm *jobPartMgr) createPipeline(ctx context.Context) { }, jpm.pacer) default: - panic(fmt.Errorf("Unrecognized FromTo: %q", fromTo.String())) + panic(fmt.Errorf("Unrecognized from-to: %q", fromTo.String())) } } } -func (jpm *jobPartMgr) SlicePool() common.ByteSlicePooler{ +func (jpm *jobPartMgr) SlicePool() common.ByteSlicePooler { return jpm.slicePool } -func (jpm *jobPartMgr) CacheLimiter() common.CacheLimiter{ +func (jpm *jobPartMgr) CacheLimiter() common.CacheLimiter { return jpm.cacheLimiter } @@ -473,9 +473,8 @@ func (jpm *jobPartMgr) SAS() (string, string) { return jpm.sourceSAS, jpm.destinationSAS } -func (jpm *jobPartMgr) localDstData() (preserveLastModifiedTime bool) { - dstData := &jpm.Plan().DstLocalData - return dstData.PreserveLastModifiedTime +func (jpm *jobPartMgr) localDstData() *JobPartPlanDstLocal { + return &jpm.Plan().DstLocalData } // Call Done when a transfer has completed its epilog; this method returns the number of transfers completed so far @@ -521,11 +520,10 @@ func (jpm *jobPartMgr) ReleaseAConnection() { func (jpm *jobPartMgr) ShouldLog(level pipeline.LogLevel) bool { return jpm.jobMgr.ShouldLog(level) } func (jpm *jobPartMgr) Log(level pipeline.LogLevel, msg string) { jpm.jobMgr.Log(level, msg) } func (jpm *jobPartMgr) Panic(err error) { jpm.jobMgr.Panic(err) } -func (jpm *jobPartMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason){ +func (jpm *jobPartMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason) { jpm.jobMgr.LogChunkStatus(id, reason) } - // TODO: Can we delete this method? // numberOfTransfersDone returns the numberOfTransfersDone_doNotUse of JobPartPlanInfo // instance in thread safe manner diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index d7ed11301..b5332f7a2 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -21,7 +21,9 @@ type IJobPartTransferMgr interface { Info() TransferInfo BlobDstData(dataFileToXfer []byte) (headers azblob.BlobHTTPHeaders, metadata azblob.Metadata) FileDstData(dataFileToXfer []byte) (headers azfile.FileHTTPHeaders, metadata azfile.Metadata) + LastModifiedTime() time.Time PreserveLastModifiedTime() (time.Time, bool) + MD5ValidationOption() common.HashValidationOption BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) //ScheduleChunk(chunkFunc chunkFunc) Context() context.Context @@ -29,7 +31,8 @@ type IJobPartTransferMgr interface { CacheLimiter() common.CacheLimiter StartJobXfer() IsForceWriteTrue() bool - ReportChunkDone() (lastChunk bool, chunksDone uint32) + ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32) + UnsafeReportChunkDone() (lastChunk bool, chunksDone uint32) TransferStatus() common.TransferStatus SetStatus(status common.TransferStatus) SetErrorCode(errorCode int32) @@ -54,6 +57,7 @@ type IJobPartTransferMgr interface { LogError(resource, context string, err error) LogTransferStart(source, destination, description string) LogChunkStatus(id common.ChunkID, reason common.WaitReason) + LogAtLevelForCurrentTransfer(level pipeline.LogLevel, msg string) common.ILogger } @@ -99,6 +103,9 @@ type jobPartTransferMgr struct { // NumberOfChunksDone determines the final cancellation or completion of a transfer atomicChunksDone uint32 + // used defensively to protect against accidental double counting + atomicCompletionIndicator uint32 + /* @Parteek removed 3/23 morning, as jeff ad equivalent // transfer chunks are put into this channel and execution engine takes chunk out of this channel. @@ -211,16 +218,25 @@ func (jptm *jobPartTransferMgr) FileDstData(dataFileToXfer []byte) (headers azfi return jptm.jobPartMgr.(*jobPartMgr).fileDstData(jptm.Info().Source, dataFileToXfer) } +// TODO refactor into something like jptm.IsLastModifiedTimeEqual() so that there is NO LastModifiedTime method and people therefore CAN'T do it wrong due to time zone +func (jptm *jobPartTransferMgr) LastModifiedTime() time.Time { + return time.Unix(0, jptm.jobPartPlanTransfer.ModifiedTime) +} + // PreserveLastModifiedTime checks for the PreserveLastModifiedTime flag in JobPartPlan of a transfer. // If PreserveLastModifiedTime is set to true, it returns the lastModifiedTime of the source. func (jptm *jobPartTransferMgr) PreserveLastModifiedTime() (time.Time, bool) { - if preserveLastModifiedTime := jptm.jobPartMgr.(*jobPartMgr).localDstData(); preserveLastModifiedTime { + if preserveLastModifiedTime := jptm.jobPartMgr.(*jobPartMgr).localDstData().PreserveLastModifiedTime; preserveLastModifiedTime { lastModifiedTime := jptm.jobPartPlanTransfer.ModifiedTime return time.Unix(0, lastModifiedTime), true } return time.Time{}, false } +func (jptm *jobPartTransferMgr) MD5ValidationOption() common.HashValidationOption { + return jptm.jobPartMgr.(*jobPartMgr).localDstData().MD5VerificationOption +} + func (jptm *jobPartTransferMgr) BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) { return jptm.jobPartMgr.BlobTiers() } @@ -234,7 +250,15 @@ func (jptm *jobPartTransferMgr) SetActionAfterLastChunk(f func()) { } // Call Done when a chunk has completed its transfer; this method returns the number of chunks completed so far -func (jptm *jobPartTransferMgr) ReportChunkDone() (lastChunk bool, chunksDone uint32) { +func (jptm *jobPartTransferMgr) ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32) { + + // Tell the id to remember that we (the jptm) have been told about its completion + // Will panic if we've already been told about its completion before. + // Why? As defensive programming, since if we accidentally counted one chunk twice, we'd complete + // before another was finish. Which would be bad + id.SetCompletionNotificationSent() + + // Do our actual processing chunksDone = atomic.AddUint32(&jptm.atomicChunksDone, 1) lastChunk = chunksDone == jptm.numChunks if lastChunk { @@ -243,6 +267,11 @@ func (jptm *jobPartTransferMgr) ReportChunkDone() (lastChunk bool, chunksDone ui return lastChunk, chunksDone } +// TODO: phase this method out. It's just here to support parts of the codebase that don't yet have chunk IDs +func (jptm *jobPartTransferMgr) UnsafeReportChunkDone() (lastChunk bool, chunksDone uint32) { + return jptm.ReportChunkDone(common.NewChunkID("", 0)) +} + // If an automatic action has been specified for after the last chunk, run it now // (Prior to introduction of this routine, individual chunkfuncs had to check the return values // of ReportChunkDone and then implement their own versions of the necessary transfer epilogue code. @@ -332,14 +361,16 @@ func (jptm *jobPartTransferMgr) failActiveTransfer(typ transferErrorCode, descri if !jptm.WasCanceled() { jptm.Cancel() status, msg := ErrorEx{err}.ErrorCodeAndString() - jptm.logTransferError(typ, jptm.Info().Source, jptm.Info().Destination, msg+" when "+descriptionOfWhereErrorOccurred, status) + requestID := ErrorEx{err}.MSRequestID() + fullMsg := fmt.Sprintf("%s. When %s. X-Ms-Request-Id: %s\n", msg, descriptionOfWhereErrorOccurred, requestID) // trailing \n to separate it better from any later, unrelated, log lines + jptm.logTransferError(typ, jptm.Info().Source, jptm.Info().Destination, fullMsg, status) jptm.SetStatus(failureStatus) jptm.SetErrorCode(int32(status)) // TODO: what are the rules about when this needs to be set, and doesn't need to be (e.g. for earlier failures)? // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { // TODO: should this really exit??? why not just log like everything else does??? We've Failed the transfer anyway.... - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } // TODO: right now the convention re cancellation seems to be that if you cancel, you MUST both call cancel AND @@ -379,7 +410,17 @@ const ( transferErrorCodeCopyFailed transferErrorCode = "COPYFAILED" ) +func (jptm *jobPartTransferMgr) LogAtLevelForCurrentTransfer(level pipeline.LogLevel, msg string) { + // order of log elements here is mirrored, with some more added, in logTransferError + fullMsg := common.URLStringExtension(jptm.Info().Source).RedactSigQueryParamForLogging() + " " + + msg + + " Dst: " + common.URLStringExtension(jptm.Info().Destination).RedactSigQueryParamForLogging() + + jptm.Log(level, fullMsg) +} + func (jptm *jobPartTransferMgr) logTransferError(errorCode transferErrorCode, source, destination, errorMsg string, status int) { + // order of log elements here is mirrored, in subset, in LogForCurrentTransfer msg := fmt.Sprintf("%v: ", errorCode) + common.URLStringExtension(source).RedactSigQueryParamForLogging() + fmt.Sprintf(" : %03d : %s\n Dst: ", status, errorMsg) + common.URLStringExtension(destination).RedactSigQueryParamForLogging() jptm.Log(pipeline.LogError, msg) @@ -402,8 +443,9 @@ func (jptm *jobPartTransferMgr) LogS2SCopyError(source, destination, errorMsg st func (jptm *jobPartTransferMgr) LogError(resource, context string, err error) { status, msg := ErrorEx{err}.ErrorCodeAndString() + MSRequestID := ErrorEx{err}.MSRequestID() jptm.Log(pipeline.LogError, - fmt.Sprintf("%s: %d: %s-%s", common.URLStringExtension(resource).RedactSigQueryParamForLogging(), status, context, msg)) + fmt.Sprintf("%s: %d: %s-%s. X-Ms-Request-Id:%s\n", common.URLStringExtension(resource).RedactSigQueryParamForLogging(), status, context, msg, MSRequestID)) } func (jptm *jobPartTransferMgr) LogTransferStart(source, destination, description string) { @@ -418,10 +460,20 @@ func (jptm *jobPartTransferMgr) Panic(err error) { jptm.jobPartMgr.Panic(err) } // Call ReportTransferDone to report when a Transfer for this Job Part has completed // TODO: I feel like this should take the status & we kill SetStatus -// TODO: also, it looks like if we accidentally call this twice, on the one jptm, it just treats that as TWO successful transfers, which is a bug func (jptm *jobPartTransferMgr) ReportTransferDone() uint32 { // In case of context leak in job part transfer manager. jptm.Cancel() + // defensive programming check, to make sure this method is not called twice for the same transfer + // (since if it was, job would count us as TWO completions, and maybe miss another transfer that + // should have been counted but wasn't) + // TODO: it would be nice if this protection was actually in jobPartMgr.ReportTransferDone, + // but that's harder to implement (would imply need for a threadsafe map there, to track + // status by transfer). So for now we are going with the check here. This is the only call + // to the jobPartManager anyway (as it Feb 2019) + if atomic.SwapUint32(&jptm.atomicCompletionIndicator, 1) != 0 { + panic("cannot report the same transfer done twice") + } + return jptm.jobPartMgr.ReportTransferDone() } diff --git a/ste/uploader-appendBlob.go b/ste/uploader-appendBlob.go index 61365652d..6cabc0dc0 100644 --- a/ste/uploader-appendBlob.go +++ b/ste/uploader-appendBlob.go @@ -38,6 +38,8 @@ type appendBlobUploader struct { pipeline pipeline.Pipeline pacer *pacer soleChunkFuncSemaphore *semaphore.Weighted + md5Channel chan []byte + creationTimeHeaders *azblob.BlobHTTPHeaders } func newAppendBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -69,6 +71,7 @@ func newAppendBlobUploader(jptm IJobPartTransferMgr, destination string, p pipel pipeline: p, pacer: pacer, soleChunkFuncSemaphore: semaphore.NewWeighted(1), + md5Channel: newMd5Channel(), }, nil } @@ -80,6 +83,10 @@ func (u *appendBlobUploader) NumChunks() uint32 { return u.numChunks } +func (u *appendBlobUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *appendBlobUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.appendBlobUrl.GetProperties(u.jptm.Context(), azblob.BlobAccessConditions{})) } @@ -93,6 +100,8 @@ func (u *appendBlobUploader) Prologue(leadingBytes []byte) { jptm.FailActiveUpload("Creating blob", err) return } + // Save headers to re-use, with same values, in epilogue + u.creationTimeHeaders = &blobHTTPHeaders } func (u *appendBlobUploader) GenerateUploadFunc(id common.ChunkID, blockIndex int32, reader common.SingleChunkReader, chunkIsWholeFile bool) chunkFunc { @@ -132,6 +141,17 @@ func (u *appendBlobUploader) GenerateUploadFunc(id common.ChunkID, blockIndex in func (u *appendBlobUploader) Epilogue() { jptm := u.jptm + + // set content MD5 (only way to do this is to re-PUT all the headers, this time with the MD5 included) + if jptm.TransferStatus() > 0 { + tryPutMd5Hash(jptm, u.md5Channel, func(md5Hash []byte) error { + epilogueHeaders := *u.creationTimeHeaders + epilogueHeaders.ContentMD5 = md5Hash + _, err := u.appendBlobUrl.SetHTTPHeaders(jptm.Context(), epilogueHeaders, azblob.BlobAccessConditions{}) + return err + }) + } + // Cleanup if jptm.TransferStatus() <= 0 { // TODO: <=0 or <0? // If the transfer status value < 0, then transfer failed with some failure diff --git a/ste/uploader-azureFiles.go b/ste/uploader-azureFiles.go index c6418a608..6aebd98bb 100644 --- a/ste/uploader-azureFiles.go +++ b/ste/uploader-azureFiles.go @@ -34,12 +34,14 @@ import ( ) type azureFilesUploader struct { - jptm IJobPartTransferMgr - fileURL azfile.FileURL - chunkSize uint32 - numChunks uint32 - pipeline pipeline.Pipeline - pacer *pacer + jptm IJobPartTransferMgr + fileURL azfile.FileURL + chunkSize uint32 + numChunks uint32 + pipeline pipeline.Pipeline + pacer *pacer + md5Channel chan []byte + creationTimeHeaders *azfile.FileHTTPHeaders // pointer so default value, nil, is clearly "wrong" and can't be used by accident } func newAzureFilesUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -68,12 +70,13 @@ func newAzureFilesUploader(jptm IJobPartTransferMgr, destination string, p pipel } return &azureFilesUploader{ - jptm: jptm, - fileURL: azfile.NewFileURL(*destURL, p), - chunkSize: chunkSize, - numChunks: numChunks, - pipeline: p, - pacer: pacer, + jptm: jptm, + fileURL: azfile.NewFileURL(*destURL, p), + chunkSize: chunkSize, + numChunks: numChunks, + pipeline: p, + pacer: pacer, + md5Channel: newMd5Channel(), }, nil } @@ -85,6 +88,10 @@ func (u *azureFilesUploader) NumChunks() uint32 { return u.numChunks } +func (u *azureFilesUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *azureFilesUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.fileURL.GetProperties(u.jptm.Context())) } @@ -107,6 +114,9 @@ func (u *azureFilesUploader) Prologue(leadingBytes []byte) { jptm.FailActiveUpload("Creating file", err) return } + + // Save headers to re-use, with same values, in epilogue + u.creationTimeHeaders = &fileHTTPHeaders } func (u *azureFilesUploader) GenerateUploadFunc(id common.ChunkID, blockIndex int32, reader common.SingleChunkReader, chunkIsWholeFile bool) chunkFunc { @@ -141,6 +151,16 @@ func (u *azureFilesUploader) GenerateUploadFunc(id common.ChunkID, blockIndex in func (u *azureFilesUploader) Epilogue() { jptm := u.jptm + // set content MD5 (only way to do this is to re-PUT all the headers, this time with the MD5 included) + if jptm.TransferStatus() > 0 { + tryPutMd5Hash(jptm, u.md5Channel, func(md5Hash []byte) error { + epilogueHeaders := *u.creationTimeHeaders + epilogueHeaders.ContentMD5 = md5Hash + _, err := u.fileURL.SetHTTPHeaders(jptm.Context(), epilogueHeaders) + return err + }) + } + // Cleanup if jptm.TransferStatus() <= 0 { // If the transfer status is less than or equal to 0 diff --git a/ste/uploader-blobFS.go b/ste/uploader-blobFS.go index 91b6b6098..8800a1a1a 100644 --- a/ste/uploader-blobFS.go +++ b/ste/uploader-blobFS.go @@ -32,12 +32,13 @@ import ( ) type blobFSUploader struct { - jptm IJobPartTransferMgr - fileURL azbfs.FileURL - chunkSize uint32 - numChunks uint32 - pipeline pipeline.Pipeline - pacer *pacer + jptm IJobPartTransferMgr + fileURL azbfs.FileURL + chunkSize uint32 + numChunks uint32 + pipeline pipeline.Pipeline + pacer *pacer + md5Channel chan []byte } func newBlobFSUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -105,12 +106,13 @@ func newBlobFSUploader(jptm IJobPartTransferMgr, destination string, p pipeline. numChunks := getNumUploadChunks(info.SourceSize, chunkSize) return &blobFSUploader{ - jptm: jptm, - fileURL: azbfs.NewFileURL(*destURL, p), - chunkSize: chunkSize, - numChunks: numChunks, - pipeline: p, - pacer: pacer, + jptm: jptm, + fileURL: azbfs.NewFileURL(*destURL, p), + chunkSize: chunkSize, + numChunks: numChunks, + pipeline: p, + pacer: pacer, + md5Channel: newMd5Channel(), }, nil } @@ -122,6 +124,11 @@ func (u *blobFSUploader) NumChunks() uint32 { return u.numChunks } +func (u *blobFSUploader) Md5Channel() chan<- []byte { + // TODO: can we support this? And when? Right now, we are returning it, but never using it ourselves + return u.md5Channel +} + func (u *blobFSUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.fileURL.GetProperties(u.jptm.Context())) } @@ -161,9 +168,15 @@ func (u *blobFSUploader) Epilogue() { // flush if jptm.TransferStatus() > 0 { - _, err := u.fileURL.FlushData(jptm.Context(), jptm.Info().SourceSize) - if err != nil { - jptm.FailActiveUpload("Flushing data", err) + md5Hash, ok := <-u.md5Channel + if ok { + _, err := u.fileURL.FlushData(jptm.Context(), jptm.Info().SourceSize, md5Hash) + if err != nil { + jptm.FailActiveUpload("Flushing data", err) + // don't return, since need cleanup below + } + } else { + jptm.FailActiveUpload("Getting hash", errNoHash) // don't return, since need cleanup below } } diff --git a/ste/uploader-blockBlob.go b/ste/uploader-blockBlob.go index e093b5fc6..40b0a5c32 100644 --- a/ste/uploader-blockBlob.go +++ b/ste/uploader-blockBlob.go @@ -44,6 +44,7 @@ type blockBlobUploader struct { leadingBytes []byte // no lock because is written before first chunk-func go routine is scheduled mu *sync.Mutex // protects the fields below blockIds []string + md5Channel chan []byte } func newBlockBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -74,6 +75,7 @@ func newBlockBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeli pacer: pacer, mu: &sync.Mutex{}, blockIds: make([]string, numChunks), + md5Channel: newMd5Channel(), }, nil } @@ -85,6 +87,10 @@ func (u *blockBlobUploader) NumChunks() uint32 { return u.numChunks } +func (u *blockBlobUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *blockBlobUploader) SetLeadingBytes(leadingBytes []byte) { u.leadingBytes = leadingBytes } @@ -151,8 +157,21 @@ func (u *blockBlobUploader) generatePutWholeBlob(id common.ChunkID, blockIndex i jptm.LogChunkStatus(id, common.EWaitReason.Body()) var err error if jptm.Info().SourceSize == 0 { + // Empty file _, err = u.blockBlobUrl.Upload(jptm.Context(), bytes.NewReader(nil), blobHttpHeader, metaData, azblob.BlobAccessConditions{}) + } else { + // File with content + + // Get the MD5 that was computed as we read the file + md5Hash, ok := <-u.md5Channel + if !ok { + jptm.FailActiveUpload("Getting hash", errNoHash) + return + } + blobHttpHeader.ContentMD5 = md5Hash + + // Upload the file body := newLiteRequestBodyPacer(reader, u.pacer) _, err = u.blockBlobUrl.Upload(jptm.Context(), body, blobHttpHeader, metaData, azblob.BlobAccessConditions{}) } @@ -172,7 +191,7 @@ func (u *blockBlobUploader) Epilogue() { blockIds := u.blockIds u.mu.Unlock() shouldPutBlockList := getPutListNeed(&u.putListIndicator) - if shouldPutBlockList == putListNeedUnknown { + if shouldPutBlockList == putListNeedUnknown && !jptm.WasCanceled() { panic("'put list' need flag was never set") } @@ -182,13 +201,20 @@ func (u *blockBlobUploader) Epilogue() { if jptm.TransferStatus() > 0 && shouldPutBlockList == putListNeeded { jptm.Log(pipeline.LogDebug, fmt.Sprintf("Conclude Transfer with BlockList %s", blockIds)) - // fetching the blob http headers with content-type, content-encoding attributes - // fetching the metadata passed with the JobPartOrder - blobHttpHeader, metaData := jptm.BlobDstData(u.leadingBytes) + md5Hash, ok := <-u.md5Channel + if ok { + // fetching the blob http headers with content-type, content-encoding attributes + // fetching the metadata passed with the JobPartOrder + blobHttpHeader, metaData := jptm.BlobDstData(u.leadingBytes) + blobHttpHeader.ContentMD5 = md5Hash - _, err := u.blockBlobUrl.CommitBlockList(jptm.Context(), blockIds, blobHttpHeader, metaData, azblob.BlobAccessConditions{}) - if err != nil { - jptm.FailActiveUpload("Committing block list", err) + _, err := u.blockBlobUrl.CommitBlockList(jptm.Context(), blockIds, blobHttpHeader, metaData, azblob.BlobAccessConditions{}) + if err != nil { + jptm.FailActiveUpload("Committing block list", err) + // don't return, since need cleanup below + } + } else { + jptm.FailActiveUpload("Getting hash", errNoHash) // don't return, since need cleanup below } } diff --git a/ste/uploader-pageBlob.go b/ste/uploader-pageBlob.go index 6c504ea09..fea651fbf 100644 --- a/ste/uploader-pageBlob.go +++ b/ste/uploader-pageBlob.go @@ -31,12 +31,14 @@ import ( ) type pageBlobUploader struct { - jptm IJobPartTransferMgr - pageBlobUrl azblob.PageBlobURL - chunkSize uint32 - numChunks uint32 - pipeline pipeline.Pipeline - pacer *pacer + jptm IJobPartTransferMgr + pageBlobUrl azblob.PageBlobURL + chunkSize uint32 + numChunks uint32 + pipeline pipeline.Pipeline + pacer *pacer + md5Channel chan []byte + creationTimeHeaders *azblob.BlobHTTPHeaders } func newPageBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -65,6 +67,7 @@ func newPageBlobUploader(jptm IJobPartTransferMgr, destination string, p pipelin numChunks: numChunks, pipeline: p, pacer: pacer, + md5Channel: newMd5Channel(), }, nil } @@ -76,6 +79,10 @@ func (u *pageBlobUploader) NumChunks() uint32 { return u.numChunks } +func (u *pageBlobUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *pageBlobUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.pageBlobUrl.GetProperties(u.jptm.Context(), azblob.BlobAccessConditions{})) } @@ -92,6 +99,8 @@ func (u *pageBlobUploader) Prologue(leadingBytes []byte) { jptm.FailActiveUpload("Creating blob", err) return } + // Save headers to re-use, with same values, in epilogue + u.creationTimeHeaders = &blobHTTPHeaders // set tier _, pageBlobTier := jptm.BlobTiers() @@ -137,6 +146,16 @@ func (u *pageBlobUploader) GenerateUploadFunc(id common.ChunkID, blockIndex int3 func (u *pageBlobUploader) Epilogue() { jptm := u.jptm + // set content MD5 (only way to do this is to re-PUT all the headers, this time with the MD5 included) + if jptm.TransferStatus() > 0 { + tryPutMd5Hash(jptm, u.md5Channel, func(md5Hash []byte) error { + epilogueHeaders := *u.creationTimeHeaders + epilogueHeaders.ContentMD5 = md5Hash + _, err := u.pageBlobUrl.SetHTTPHeaders(jptm.Context(), epilogueHeaders, azblob.BlobAccessConditions{}) + return err + }) + } + // Cleanup if jptm.TransferStatus() <= 0 { // TODO: <=0 or <0? deletionContext, cancelFn := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/ste/uploader.go b/ste/uploader.go index 58ba4318f..7de937d2a 100644 --- a/ste/uploader.go +++ b/ste/uploader.go @@ -21,6 +21,7 @@ package ste import ( + "errors" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" ) @@ -58,10 +59,35 @@ type uploader interface { // or post-failure processing otherwise. Implementations should // use jptm.FailActiveUpload if anything fails during the epilogue. Epilogue() + + // Md5Channel returns the channel on which localToRemote should send the MD5 hash to the uploader + Md5Channel() chan<- []byte } type uploaderFactory func(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) +func newMd5Channel() chan []byte { + return make(chan []byte, 1) // must be buffered, so as not to hold up the goroutine running localToRemote (which needs to start on the NEXT file after finishing its current one) +} + +// Tries to set the MD5 hash using the given function +// Fails the upload if any error happens. +// This should be used only by those uploads that require a separate operation to PUT the hash at the end. +// Others, such as the block blob uploader piggyback their MD5 setting on other calls, and so won't use this. +func tryPutMd5Hash(jptm IJobPartTransferMgr, md5Channel <-chan []byte, worker func(hash []byte) error) { + md5Hash, ok := <-md5Channel + if ok { + err := worker(md5Hash) + if err != nil { + jptm.FailActiveUpload("Setting hash", err) + } + } else { + jptm.FailActiveUpload("Setting hash", errNoHash) + } +} + +var errNoHash = errors.New("no hash computed") + func getNumUploadChunks(fileSize int64, chunkSize uint32) uint32 { numChunks := uint32(1) // for uploads, we always map zero-size files to ONE (empty) chunk if fileSize > 0 { @@ -86,7 +112,7 @@ func createChunkFunc(setDoneStatusOnExit bool, jptm IJobPartTransferMgr, id comm return func(workerId int) { // BEGIN standard prefix that all chunk funcs need - defer jptm.ReportChunkDone() // whether successful or failed, it's always "done" and we must always tell the jptm + defer jptm.ReportChunkDone(id) // whether successful or failed, it's always "done" and we must always tell the jptm jptm.OccupyAConnection() // TODO: added the two operations for debugging purpose. remove later defer jptm.ReleaseAConnection() diff --git a/ste/xfer-URLToBlob.go b/ste/xfer-URLToBlob.go index ee3d19c4e..48c052ac4 100644 --- a/ste/xfer-URLToBlob.go +++ b/ste/xfer-URLToBlob.go @@ -114,7 +114,7 @@ func URLToBlob(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } } else { @@ -229,7 +229,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd if bbc.jptm.ShouldLog(pipeline.LogDebug) { bbc.jptm.Log(pipeline.LogDebug, fmt.Sprintf("Transfer cancelled. not picking up chunk %d", chunkId)) } - if lastChunk, _ := bbc.jptm.ReportChunkDone(); lastChunk { + if lastChunk, _ := bbc.jptm.UnsafeReportChunkDone(); lastChunk { if bbc.jptm.ShouldLog(pipeline.LogDebug) { bbc.jptm.Log(pipeline.LogDebug, "Finalizing transfer cancellation") } @@ -262,11 +262,11 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } - if lastChunk, _ := bbc.jptm.ReportChunkDone(); lastChunk { + if lastChunk, _ := bbc.jptm.UnsafeReportChunkDone(); lastChunk { if bbc.jptm.ShouldLog(pipeline.LogDebug) { bbc.jptm.Log(pipeline.LogDebug, "Finalizing transfer cancellation") } @@ -276,7 +276,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd } // step 4: check if this is the last chunk - if lastChunk, _ := bbc.jptm.ReportChunkDone(); lastChunk { + if lastChunk, _ := bbc.jptm.UnsafeReportChunkDone(); lastChunk { // If the transfer gets cancelled before the putblock list if bbc.jptm.WasCanceled() { transferDone() @@ -299,7 +299,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } return } diff --git a/ste/xfer-deleteBlob.go b/ste/xfer-deleteBlob.go index 1a2b42f99..c3509373e 100644 --- a/ste/xfer-deleteBlob.go +++ b/ste/xfer-deleteBlob.go @@ -52,7 +52,9 @@ func DeleteBlobPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if strErr.Response().StatusCode == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + errMsg := fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()) + jptm.Log(pipeline.LogError, errMsg) + common.GetLifecycleMgr().Error(errMsg) } } else { transferDone(common.ETransferStatus.Failed(), err) diff --git a/ste/xfer-deletefile.go b/ste/xfer-deleteFile.go similarity index 89% rename from ste/xfer-deletefile.go rename to ste/xfer-deleteFile.go index 22cdf04d9..325782e7f 100644 --- a/ste/xfer-deletefile.go +++ b/ste/xfer-deleteFile.go @@ -52,7 +52,9 @@ func DeleteFilePrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if strErr.Response().StatusCode == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + errMsg := fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()) + jptm.Log(pipeline.LogError, errMsg) + common.GetLifecycleMgr().Error(errMsg) } } else { transferDone(common.ETransferStatus.Failed(), err) diff --git a/ste/xfer-localToRemote.go b/ste/xfer-localToRemote.go index 9050b44df..850c262dc 100644 --- a/ste/xfer-localToRemote.go +++ b/ste/xfer-localToRemote.go @@ -21,6 +21,8 @@ package ste import ( + "crypto/md5" + "errors" "fmt" "os" @@ -48,13 +50,14 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, jptm.ReportTransferDone() return } - // step 2b. Read chunk size and count from the uploader (since it may have applied its own defaults and/or calculations to produce these values - chunkSize := ul.ChunkSize() - numChunks := ul.NumChunks() + md5Channel := ul.Md5Channel() + defer close(md5Channel) // never leave receiver hanging, waiting for a result, even if we fail here + + // step 2b. Check chunk size and count from the uploader (it may have applied its own defaults and/or calculations to produce these values if jptm.ShouldLog(pipeline.LogInfo) { - jptm.LogTransferStart(info.Source, info.Destination, fmt.Sprintf("Specified chunk size %d", chunkSize)) + jptm.LogTransferStart(info.Source, info.Destination, fmt.Sprintf("Specified chunk size %d", ul.ChunkSize())) } - if numChunks == 0 { + if ul.NumChunks() == 0 { panic("must always schedule one chunk, even if file is empty") // this keeps our code structure simpler, by using a dummy chunk for empty files } @@ -92,6 +95,20 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, } defer srcFile.Close() // we read all the chunks in this routine, so can close the file at the end + i, err := os.Stat(info.Source) + if err != nil { + jptm.LogUploadError(info.Source, info.Destination, "Couldn't stat source-"+err.Error(), 0) + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() + return + } + if i.ModTime() != jptm.LastModifiedTime() { + jptm.LogUploadError(info.Source, info.Destination, "File modified since transfer scheduled", 0) + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() + return + } + // ***** // Error-handling rules change here. // ABOVE this point, we end the transfer using the code as shown above @@ -102,7 +119,7 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // ****** // step 5: tell jptm what to expect, and how to clean up at the end - jptm.SetNumberOfChunks(numChunks) + jptm.SetNumberOfChunks(ul.NumChunks()) jptm.SetActionAfterLastChunk(func() { epilogueWithCleanupUpload(jptm, ul) }) // TODO: currently, the epilogue will only run if the number of completed chunks = numChunks. @@ -114,20 +131,30 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // eventually reach numChunks, since we have no better short-term alternative. // Step 5: Go through the file and schedule chunk messages to upload each chunk - // As we do this, we force preload of each chunk to memory, and we wait (block) - // here if the amount of preloaded data gets excessive. That's OK to do, - // because if we already have that much data preloaded (and scheduled for sending in - // chunks) then we don't need to schedule any more chunks right now, so the blocking - // is harmless (and a good thing, to avoid excessive RAM usage). - // To take advantage of the good sequential read performance provided by many file systems, - // we work sequentially through the file here. + scheduleUploadChunks(jptm, info.Source, srcFile, fileSize, ul, sourceFileFactory, md5Channel) +} + +// Schedule all the upload chunks. +// As we do this, we force preload of each chunk to memory, and we wait (block) +// here if the amount of preloaded data gets excessive. That's OK to do, +// because if we already have that much data preloaded (and scheduled for sending in +// chunks) then we don't need to schedule any more chunks right now, so the blocking +// is harmless (and a good thing, to avoid excessive RAM usage). +// To take advantage of the good sequential read performance provided by many file systems, +// and to be able to compute an MD5 hash for the file, we work sequentially through the file here. +func scheduleUploadChunks(jptm IJobPartTransferMgr, srcName string, srcFile common.CloseableReaderAt, fileSize int64, ul uploader, sourceFileFactory common.ChunkReaderSourceFactory, md5Channel chan<- []byte) { + chunkSize := ul.ChunkSize() + numChunks := ul.NumChunks() context := jptm.Context() slicePool := jptm.SlicePool() cacheLimiter := jptm.CacheLimiter() + chunkCount := int32(0) + md5Hasher := md5.New() + safeToUseHash := true for startIndex := int64(0); startIndex < fileSize || isDummyChunkInEmptyFile(startIndex, fileSize); startIndex += int64(chunkSize) { - id := common.ChunkID{Name: info.Source, OffsetInFile: startIndex} + id := common.NewChunkID(srcName, startIndex) adjustedChunkSize := int64(chunkSize) // compute actual size of the chunk @@ -140,10 +167,23 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // of the file read later (when doing a retry) // BTW, the reader we create here just works with a single chuck. (That's in contrast with downloads, where we have // to use an object that encompasses the whole file, so that it can put the chunks back into order. We don't have that requirement here.) - chunkReader := common.NewSingleChunkReader(context, sourceFileFactory, id, adjustedChunkSize, jptm, slicePool, cacheLimiter) + chunkReader := common.NewSingleChunkReader(context, sourceFileFactory, id, adjustedChunkSize, jptm, jptm, slicePool, cacheLimiter) // Wait until we have enough RAM, and when we do, prefetch the data for this chunk. - chunkReader.TryBlockingPrefetch(srcFile) + chunkDataError := chunkReader.BlockingPrefetch(srcFile, false) + + // Add the bytes to the hash + // NOTE: if there is a retry on this chunk later (a 503 from Service) our current implementation of singleChunkReader + // (as at Jan 2019) will re-read from the disk. If that part of the file has been updated by another process, + // that means it will not longer match the hash we set here. That would be bad. So we rely on logic + // elsewhere in our upload code to avoid/fail or retry such transfers. + // TODO: move the above note to the place where we implement the avoid/fail/retry and refer to that in a comment + // on the retry file-re-read logic + if chunkDataError == nil { + chunkReader.WriteBufferTo(md5Hasher) + } else { + safeToUseHash = false // because we've missed a chunk + } // If this is the the very first chunk, do special init steps if startIndex == 0 { @@ -158,15 +198,27 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // schedule the chunk job/msg jptm.LogChunkStatus(id, common.EWaitReason.WorkerGR()) - isWholeFile := numChunks == 1 - jptm.ScheduleChunks(ul.GenerateUploadFunc(id, chunkCount, chunkReader, isWholeFile)) + var cf chunkFunc + if chunkDataError == nil { + isWholeFile := numChunks == 1 + cf = ul.GenerateUploadFunc(id, chunkCount, chunkReader, isWholeFile) + } else { + _ = chunkReader.Close() + // Our jptm logic currently requires us to schedule every chunk, even if we know there's an error, + // so we schedule a func that will just fail with the given error + cf = createUploadChunkFunc(jptm, id, func() { jptm.FailActiveUpload("chunk data read", chunkDataError) }) + } + jptm.ScheduleChunks(cf) chunkCount += 1 } - // sanity check to verify the number of chunks scheduled if chunkCount != int32(numChunks) { - panic(fmt.Errorf("difference in the number of chunk calculated %v and actual chunks scheduled %v for src %s of size %v", numChunks, chunkCount, info.Source, fileSize)) + panic(fmt.Errorf("difference in the number of chunk calculated %v and actual chunks scheduled %v for src %s of size %v", numChunks, chunkCount, srcName, fileSize)) + } + // provide the hash that we computed + if safeToUseHash { + md5Channel <- md5Hasher.Sum(nil) } } @@ -179,6 +231,17 @@ func isDummyChunkInEmptyFile(startIndex int64, fileSize int64) bool { // depend on the destination type func epilogueWithCleanupUpload(jptm IJobPartTransferMgr, ul uploader) { + if jptm.TransferStatus() > 0 { + // Stat the file again to see if it was changed during transfer. If it was, mark the transfer as failed. + i, err := os.Stat(jptm.Info().Source) + if err != nil { + jptm.FailActiveUpload("epilogueWithCleanupUpload", err) + } + if i.ModTime() != jptm.LastModifiedTime() { + jptm.FailActiveUpload("epilogueWithCleanupUpload", errors.New("source modified during transfer")) + } + } + ul.Epilogue() // TODO: finalize and wrap in functions whether 0 is included or excluded in status comparisons diff --git a/ste/xfer-remoteToLocal.go b/ste/xfer-remoteToLocal.go index fd769deb3..2609a9657 100644 --- a/ste/xfer-remoteToLocal.go +++ b/ste/xfer-remoteToLocal.go @@ -81,7 +81,7 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, if err != nil { jptm.LogDownloadError(info.Source, info.Destination, "File Creation Error "+err.Error(), 0) jptm.SetStatus(common.ETransferStatus.Failed()) - epilogueWithCleanupDownload(jptm, nil, nil) + epilogueWithCleanupDownload(jptm, nil, nil) // use standard epilogue for consistency return } // TODO: Question: do we need to Stat the file, to check its size, after explicitly making it with the desired size? @@ -118,7 +118,8 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, chunkLogger, dstFile, numChunks, - MaxRetryPerDownloadBody) + MaxRetryPerDownloadBody, + jptm.MD5ValidationOption()) // step 5c: tell jptm what to expect, and how to clean up at the end jptm.SetNumberOfChunks(numChunks) @@ -135,7 +136,7 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, chunkCount := uint32(0) for startIndex := int64(0); startIndex < fileSize; startIndex += downloadChunkSize { - id := common.ChunkID{Name: info.Destination, OffsetInFile: startIndex} + id := common.NewChunkID(info.Destination, startIndex) adjustedChunkSize := downloadChunkSize // compute exact size of the chunk @@ -170,22 +171,30 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, func epilogueWithCleanupDownload(jptm IJobPartTransferMgr, activeDstFile *os.File, cw common.ChunkedFileWriter) { info := jptm.Info() - if activeDstFile != nil { + haveNonEmptyFile := activeDstFile != nil + if haveNonEmptyFile { + // wait until all received chunks are flushed out - _, flushError := cw.Flush(jptm.Context()) // todo: use, and check the MD5 hash returned here + md5OfFileAsWritten, flushError := cw.Flush(jptm.Context()) + closeErr := activeDstFile.Close() // always try to close if, even if flush failed + if flushError != nil { + jptm.FailActiveDownload("Flushing file", flushError) + } + if closeErr != nil { + jptm.FailActiveDownload("Closing file", closeErr) + } - // Close file - fileCloseErr := activeDstFile.Close() // always try to close if, even if flush failed - if (flushError != nil || fileCloseErr != nil) && !jptm.TransferStatus().DidFail() { - // it WAS successful up to now, but the file flush/closing failed. - message := "" - if flushError != nil { - message = "File Flush Error " + flushError.Error() - } else { - message = "File Closure Error " + fileCloseErr.Error() + // Check MD5 (but only if file was fully flushed and saved - else no point and may not have actualAsSaved hash anyway) + if !jptm.TransferStatus().DidFail() { + comparison := md5Comparer{ + expected: info.SrcHTTPHeaders.ContentMD5, // the MD5 that came back from Service when we enumerated the source + actualAsSaved: md5OfFileAsWritten, + validationOption: jptm.MD5ValidationOption(), + logger: jptm} + err := comparison.Check() + if err != nil { + jptm.FailActiveDownload("Checking MD5 hash", err) } - jptm.LogDownloadError(info.Source, info.Destination, message, 0) - jptm.SetStatus(common.ETransferStatus.Failed()) } } @@ -199,9 +208,8 @@ func epilogueWithCleanupDownload(jptm IJobPartTransferMgr, activeDstFile *os.Fil err := os.Chtimes(jptm.Info().Destination, lastModifiedTime, lastModifiedTime) if err != nil { jptm.LogError(info.Destination, "Changing Modified Time ", err) - return - } - if jptm.ShouldLog(pipeline.LogInfo) { + // do NOT return, since final status and cleanup logging still to come + } else { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" Preserved Modified Time for %s", info.Destination)) } } diff --git a/ste/xfer.go b/ste/xfer.go index 6acad2cad..50c17c81e 100644 --- a/ste/xfer.go +++ b/ste/xfer.go @@ -96,7 +96,7 @@ func computeJobXfer(fromTo common.FromTo, blobType common.BlobType) newJobXfer { case common.EFromTo.FileBlob(): return URLToBlob } - panic(fmt.Errorf("Unrecognized FromTo: %q", fromTo.String())) + panic(fmt.Errorf("Unrecognized from-to: %q", fromTo.String())) } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/testSuite/cmd/testblobFS.go b/testSuite/cmd/testblobFS.go index b31671013..ad394bacc 100644 --- a/testSuite/cmd/testblobFS.go +++ b/testSuite/cmd/testblobFS.go @@ -8,7 +8,6 @@ import ( "net/url" "os" "path/filepath" - "strconv" "strings" "github.com/Azure/azure-storage-azcopy/azbfs" @@ -96,11 +95,7 @@ func (tbfsc TestBlobFSCommand) verifyRemoteFile() { os.Exit(1) } // get the size of the downloaded file - downloadedLength, err := strconv.ParseInt(dResp.ContentLength(), 10, 64) - if err != nil { - fmt.Println("error converting the content length to int64. failed with error ", err.Error()) - os.Exit(1) - } + downloadedLength := dResp.ContentLength() // open the local file f, err := os.Open(tbfsc.Object) diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index 529b26bac..9914f40f2 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -9,6 +9,7 @@ from test_blobfs_download_sharedkey import * from test_blobfs_download_oauth import * from test_blob_piping import * +from test_blob_sync import * from test_service_to_service_copy import * import glob, os import configparser @@ -16,6 +17,7 @@ import sys import unittest + def parse_config_file_set_env(): config = configparser.RawConfigParser() files_read = config.read('../test_suite_config.ini') @@ -163,6 +165,7 @@ def main(): init() test_class_to_run = [BlobPipingTests, + Blob_Sync_User_Scenario, Block_Upload_User_Scenarios, Blob_Download_User_Scenario, PageBlob_Upload_User_Scenarios, diff --git a/testSuite/scripts/test_blob_download.py b/testSuite/scripts/test_blob_download.py index af40756d4..d46311b89 100644 --- a/testSuite/scripts/test_blob_download.py +++ b/testSuite/scripts/test_blob_download.py @@ -152,137 +152,6 @@ def test_blob_download_with_special_characters(self): result = util.Command("testBlob").add_arguments(filepath).add_arguments(resource_url).execute_azcopy_verify() self.assertTrue(result) - def test_sync_blob_download_without_wildcards(self): - # created a directory and created 10 files inside the directory - dir_name = "sync_download_without_wildcards" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # upload the directory - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # execute the validator and verify the downloaded dir - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # sync the source and destination - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - - def test_sync_blob_download_with_wildcards(self): - # created a directory and created 10 files inside the directory - dir_name = "sync_download_with_wildcards" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # upload the directory - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # execute the validator and verify the downloaded dir - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # add "*" at the end of dir sas - # since both the source and destination are in sync, it will fail - dir_sas = util.append_text_path_resource_sas(dir_sas, "*") - # sync the source and destination - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - subdir1 = os.path.join(dir_name, "subdir1") - subdir1_file_path = util.create_test_n_files(1024, 10, subdir1) - - subdir2 = os.path.join(dir_name, "subdir2") - subdir2_file_path = util.create_test_n_files(1024, 10, subdir2) - - # upload the directory - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # Download the directory to match the blob modified time - result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # sync the source and destination - # add extra wildcards - # since source and destination both are in sync, it will will not perform any sync.s - dir_sas = util.append_text_path_resource_sas(dir_sas, "*/*.txt") - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # delete 5 files inside each sub-directories locally - for r in range(5, 10): - filename = "test101024_" + str(r) + ".txt" - filepath = os.path.join(subdir1_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file ' + filepath) - filepath = os.path.join(subdir2_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file ' + filepath) - # 10 files have been deleted inside the sub-dir - # sync remote to local - # 10 files will be downloaded - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output","json").\ - execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - # Number of Expected Transfer should be 10 since 10 files were deleted - self.assertEquals(x.CopyTransfersCompleted, 10) - self.assertEquals(x.CopyTransfersFailed, 0) - # test_download_1kb_blob verifies the download of 1Kb blob using azcopy. def test_download_1kb_blob_with_oauth(self): self.util_test_download_1kb_blob_with_oauth() diff --git a/testSuite/scripts/test_blob_sync.py b/testSuite/scripts/test_blob_sync.py new file mode 100644 index 000000000..7542fe4ca --- /dev/null +++ b/testSuite/scripts/test_blob_sync.py @@ -0,0 +1,85 @@ +import json +import os +import shutil +import time +import urllib +from collections import namedtuple +import utility as util +import unittest + + +# Temporary tests (mostly copy-pasted from blob tests) to guarantee simple sync scenarios still work +# TODO Replace with better tests in the future +class Blob_Sync_User_Scenario(unittest.TestCase): + + def test_sync_single_blob(self): + # create file of size 1KB. + filename = "test_1kb_blob_sync.txt" + file_path = util.create_test_file(filename, 1024) + blob_path = util.get_resource_sas(filename) + + # Upload 1KB file using azcopy. + src = file_path + dest = blob_path + result = util.Command("cp").add_arguments(src).add_arguments(dest). \ + add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + # Verifying the uploaded blob. + # the resource local path should be the first argument for the azcopy validator. + # the resource sas should be the second argument for azcopy validator. + resource_url = util.get_resource_sas(filename) + result = util.Command("testBlob").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify() + self.assertTrue(result) + + # Sync 1KB file to local using azcopy. + src = blob_path + dest = file_path + result = util.Command("sync").add_arguments(src).add_arguments(dest). \ + add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + # Sync 1KB file to blob using azcopy. + # reset local file lmt first + util.create_test_file(filename, 1024) + src = file_path + dest = blob_path + result = util.Command("sync").add_arguments(src).add_arguments(dest). \ + add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + def test_sync_entire_directory(self): + dir_name = "dir_sync_test" + dir_path = util.create_test_n_files(1024, 10, dir_name) + + # create sub-directory inside directory + sub_dir_name = os.path.join(dir_name, "sub_dir_sync_test") + util.create_test_n_files(1024, 10, sub_dir_name) + + # upload the directory with 20 files + # upload the directory + # execute azcopy command + result = util.Command("copy").add_arguments(dir_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + # execute the validator. + vdir_sas = util.get_resource_sas(dir_name) + result = util.Command("testBlob").add_arguments(dir_path).add_arguments(vdir_sas). \ + add_flags("is-object-dir", "true").execute_azcopy_verify() + self.assertTrue(result) + + # sync to local + src = vdir_sas + dst = dir_path + result = util.Command("sync").add_arguments(src).add_arguments(dst).add_flags("log-level", "info")\ + .execute_azcopy_copy_command() + self.assertTrue(result) + + # sync back to blob after recreating the files + util.create_test_n_files(1024, 10, sub_dir_name) + src = dir_path + dst = vdir_sas + result = util.Command("sync").add_arguments(src).add_arguments(dst).add_flags("log-level", "info") \ + .execute_azcopy_copy_command() + self.assertTrue(result) diff --git a/testSuite/scripts/test_upload_block_blob.py b/testSuite/scripts/test_upload_block_blob.py index 330eff9f6..a91752df1 100644 --- a/testSuite/scripts/test_upload_block_blob.py +++ b/testSuite/scripts/test_upload_block_blob.py @@ -580,191 +580,6 @@ def test_download_blob_exclude_flag(self): self.assertEquals(x.TransfersCompleted, 10) self.assertEquals(x.TransfersFailed, 0) - def test_sync_local_to_blob_without_wildCards(self): - # create 10 files inside the dir 'sync_local_blob' - dir_name = "sync_local_blob" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # create sub-dir inside dir sync_local_blob - # create 10 files inside the sub-dir of size 1024 - sub_dir_name = os.path.join(dir_name, "sub_dir_sync_local_blob") - sub_dir_n_file_path = util.create_test_n_files(1024, 10, sub_dir_name) - - # uploading the directory with 20 files in it. - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - # execute the validator and validating the uploaded directory. - destination = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # execute a sync command - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - # since source and destination both are in sync, there should no sync and the azcopy should exit with error code - self.assertTrue(result) - try: - shutil.rmtree(sub_dir_n_file_path) - except: - self.fail('error deleting the directory' + sub_dir_n_file_path) - - # deleted entire sub-dir inside the dir created above - # sync between source and destination should delete the sub-dir on container - # number of successful transfer should be equal to 10 - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - - # Number of Expected Transfer should be 10 since sub-dir is to exclude which has 10 files in it. - self.assertEquals(x.DeleteTransfersCompleted, 10) - self.assertEquals(x.DeleteTransfersFailed, 0) - - # delete 5 files inside the directory - for r in range(5, 10): - filename = "test101024_" + str(r) + ".txt" - filepath = os.path.join(dir_n_files_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file ' + filepath) - - # sync between source and destination should delete the deleted files on container - # number of successful transfer should be equal to 5 - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - - # Number of Expected Transfer should be 10 since 10 files were deleted - self.assertEquals(x.DeleteTransfersCompleted, 5) - self.assertEquals(x.DeleteTransfersFailed, 0) - - # change the modified time of file - # perform the sync - # expected number of transfer is 1 - filepath = os.path.join(dir_n_files_path, "test101024_0.txt") - st = os.stat(filepath) - atime = st[ST_ATIME] # access time - mtime = st[ST_MTIME] # modification time - new_mtime = mtime + (4 * 3600) # new modification time - os.utime(filepath, (atime, new_mtime)) - # sync source to destination - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - # Number of Expected Transfer should be 1 since 1 file's modified time was changed - self.assertEquals(x.CopyTransfersCompleted, 1) - self.assertEquals(x.CopyTransfersFailed, 0) - - def test_sync_local_to_blob_with_wildCards(self): - # create 10 files inside the dir 'sync_local_blob' - dir_name = "sync_local_blob_wc" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # create sub-dir inside dir sync_local_blob_wc - # create 10 files inside the sub-dir of size 1024 - sub_dir_1 = os.path.join(dir_name, "sub_dir_1") - sub_dir1_n_file_path = util.create_test_n_files(1024, 10, sub_dir_1) - - # create sub-dir inside dir sync_local_blob_wc - sub_dir_2 = os.path.join(dir_name, "sub_dir_2") - sub_dir2_n_file_path = util.create_test_n_files(1024, 10, sub_dir_2) - - # uploading the directory with 30 files in it. - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator and validating the uploaded directory. - destination = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # add wildcard at the end of dirpath - dir_n_files_path_wcard = os.path.join(dir_n_files_path, "*") - # execute a sync command - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("sync").add_arguments(dir_n_files_path_wcard).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - # since source and destination both are in sync, there should no sync and the azcopy should exit with error code - self.assertTrue(result) - - # sync all the files the ends with .txt extension inside all sub-dirs inside inside - # sd_dir_n_files_path_wcard is in format dir/*/*.txt - sd_dir_n_files_path_wcard = os.path.join(dir_n_files_path_wcard, "*.txt") - result = util.Command("sync").add_arguments(sd_dir_n_files_path_wcard).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - # since source and destination both are in sync, there should no sync and the azcopy should exit with error code - self.assertTrue(result) - - # remove 5 files inside both the sub-directories - for r in range(5, 10): - filename = "test101024_" + str(r) + ".txt" - filepath = os.path.join(sub_dir1_n_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file '+ filepath) - filepath = os.path.join(sub_dir2_n_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file '+ filepath) - # sync all the files the ends with .txt extension inside all sub-dirs inside inside - # since 5 files inside each sub-dir are deleted, sync will have total 10 transfer - # 10 files will deleted from container - sd_dir_n_files_path_wcard = os.path.join(dir_n_files_path_wcard, "*.txt") - result = util.Command("sync").add_arguments(sd_dir_n_files_path_wcard).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - # Number of Expected Transfer should be 10 since 10 files were deleted - self.assertEquals(x.DeleteTransfersCompleted, 10) - self.assertEquals(x.DeleteTransfersFailed, 0) - - def test_0KB_blob_upload(self): # Creating a single File Of size 0 KB filename = "test0KB.txt" diff --git a/testSuite/scripts/test_upload_page_blob.py b/testSuite/scripts/test_upload_page_blob.py index 6bc723707..dfb465c42 100644 --- a/testSuite/scripts/test_upload_page_blob.py +++ b/testSuite/scripts/test_upload_page_blob.py @@ -18,7 +18,7 @@ def util_test_page_blob_upload_1mb(self, use_oauth=False): dest_validate = util.get_resource_from_oauth_container_validate(file_name) result = util.Command("copy").add_arguments(file_path).add_arguments(dest).add_flags("log-level", "info"). \ - add_flags("block-size", "4194304").add_flags("blobType","PageBlob").execute_azcopy_copy_command() + add_flags("block-size", "4194304").add_flags("blob-type","PageBlob").execute_azcopy_copy_command() self.assertTrue(result) # execute validator. @@ -46,7 +46,7 @@ def test_page_range_for_complete_sparse_file(self): # execute azcopy page blob upload. destination_sas = util.get_resource_sas(file_name) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \ - add_flags("block-size", "4194304").add_flags("blobType","PageBlob").execute_azcopy_copy_command() + add_flags("block-size", "4194304").add_flags("blob-type","PageBlob").execute_azcopy_copy_command() self.assertTrue(result) # execute validator. @@ -66,7 +66,7 @@ def test_page_blob_upload_partial_sparse_file(self): # execute azcopy pageblob upload. destination_sas = util.get_resource_sas(file_name) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \ - add_flags("block-size", "4194304").add_flags("blobType","PageBlob").execute_azcopy_copy_command() + add_flags("block-size", "4194304").add_flags("blob-type","PageBlob").execute_azcopy_copy_command() self.assertTrue(result) # number of page range for partial sparse created above will be (size/2) @@ -85,7 +85,7 @@ def test_set_page_blob_tier(self): destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P10").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P10").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -99,7 +99,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P20").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P20").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -113,7 +113,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P30").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P30").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -127,7 +127,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P4").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P4").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -141,7 +141,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P40").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P40").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -155,7 +155,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P50").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P50").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -169,7 +169,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P6").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P6").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. diff --git a/testSuite/scripts/utility.py b/testSuite/scripts/utility.py index 4bb77c3d0..1c0b6a286 100644 --- a/testSuite/scripts/utility.py +++ b/testSuite/scripts/utility.py @@ -8,6 +8,7 @@ import random import json from pathlib import Path +from collections import namedtuple # Command Class is used to create azcopy commands and validator commands. @@ -84,7 +85,7 @@ def process_oauth_command( cmd, fromTo=""): if fromTo!="": - cmd.add_flags("fromTo", fromTo) + cmd.add_flags("from-to", fromTo) # api executes the clean command on validator which deletes all the contents of the container. def clean_test_container(container): @@ -664,7 +665,9 @@ def parseAzcopyOutput(s): final_output = final_output + '\n' + line else: final_output = line - return final_output + + x = json.loads(final_output, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + return x.MessageContent def get_resource_name(prefix=''): return prefix + str(uuid.uuid4()).replace('-', '')