Update v2017_04_17 SDK to the 0.37.1 packages.

Additionally, add a script to automate the updating process.

Author: troydai

README.rst

@@ -17,6 +17,11 @@ Handles multi-API versions of Azure Storage Data Plane originally from https://g
 Change Log
 ----------
 
+0.1.7
++++++
+* Upgrade:
+    - 2017-04-17 (from 0.37.0 to 0.37.1)
+
 0.1.6
 +++++
 * Integrate the latest Python Storage SDK as well as the CosmosDB table SDK

azure/multiapi/storage/v2017_04_17/blob/_constants.py

@@ -14,7 +14,10 @@
 # --------------------------------------------------------------------------
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '0.36.0'
+__version__ = '0.37.1'
 
 # x-ms-version for storage service.
 X_MS_VERSION = '2017-04-17'
+
+# internal configurations, should not be changed
+_LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024

azure/multiapi/storage/v2017_04_17/blob/_deserialization.py

@@ -244,7 +244,9 @@ def _convert_xml_to_containers(response):
     'CopyCompletionTime': ('copy', 'completion_time', _to_str),
     'CopyStatusDescription': ('copy', 'status_description', _to_str),
     'AccessTier': (None, 'blob_tier', _to_str),
-    'ArchiveStatus': (None, 'rehydration_status', _to_str)
+    'AccessTierChangeTime': (None, 'blob_tier_change_time', parser.parse),
+    'AccessTierInferred': (None, 'blob_tier_inferred', _bool),
+    'ArchiveStatus': (None, 'rehydration_status', _to_str),
 }
 
 
@@ -281,6 +283,8 @@ def _convert_xml_to_blob_list(response):
             <CopyCompletionTime>datetime</CopyCompletionTime>
             <CopyStatusDescription>error string</CopyStatusDescription>
             <AccessTier>P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot</AccessTier>
+            <AccessTierChangeTime>date-time-value</AccessTierChangeTime>
+            <AccessTierInferred>true</AccessTierInferred>
           </Properties>
           <Metadata>   
             <Name>value</Name>

azure/multiapi/storage/v2017_04_17/blob/_upload_chunking.py

@@ -27,6 +27,9 @@
     _get_blob_encryptor_and_padder,
 )
 from .models import BlobBlock
+from ._constants import (
+    _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE
+)
 
 
 def _upload_blob_chunks(blob_service, container_name, blob_name,
@@ -342,6 +345,7 @@ def __init__(self, wrapped_stream, stream_begin_index, length, lockObj):
         # derivations of io.IOBase and thus do not implement seekable().
         # Python > 3.0: file-like objects created with open() are derived from io.IOBase.
         try:
+            # only the main thread runs this, so there's no need grabbing the lock
             wrapped_stream.seek(0, SEEK_CUR)
         except:
             raise ValueError("Wrapped stream must support seek().")
@@ -351,9 +355,14 @@ def __init__(self, wrapped_stream, stream_begin_index, length, lockObj):
         self._position = 0
         self._stream_begin_index = stream_begin_index
         self._length = length
-        self._count = 0
         self._buffer = BytesIO()
-        self._read_buffer_size = 4 * 1024 * 1024
+
+        # we must avoid buffering more than necessary, and also not use up too much memory
+        # so the max buffer size is capped at 4MB
+        self._max_buffer_size = length if length < _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE \
+            else _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE
+        self._current_buffer_start = 0
+        self._current_buffer_size = 0
 
     def __len__(self):
         return self._length
@@ -382,35 +391,45 @@ def read(self, n):
         if n is 0 or self._buffer.closed:
             return b''
 
-        # attempt first read from the read buffer
+        # attempt first read from the read buffer and update position
         read_buffer = self._buffer.read(n)
         bytes_read = len(read_buffer)
         bytes_remaining = n - bytes_read
+        self._position += bytes_read
 
         # repopulate the read buffer from the underlying stream to fulfill the request
         # ensure the seek and read operations are done atomically (only if a lock is provided)
         if bytes_remaining > 0:
             with self._buffer:
+                # either read in the max buffer size specified on the class
+                # or read in just enough data for the current block/sub stream
+                current_max_buffer_size = min(self._max_buffer_size, self._length - self._position)
+
                 # lock is only defined if max_connections > 1 (parallel uploads)
                 if self._lock:
                     with self._lock:
-                        # reposition the underlying stream to match the start of the substream
+                        # reposition the underlying stream to match the start of the data to read
                         absolute_position = self._stream_begin_index + self._position
                         self._wrapped_stream.seek(absolute_position, SEEK_SET)
                         # If we can't seek to the right location, our read will be corrupted so fail fast.
                         if self._wrapped_stream.tell() != absolute_position:
                             raise IOError("Stream failed to seek to the desired location.")
-                        buffer_from_stream = self._wrapped_stream.read(self._read_buffer_size)
+                        buffer_from_stream = self._wrapped_stream.read(current_max_buffer_size)
                 else:
-                    buffer_from_stream = self._wrapped_stream.read(self._read_buffer_size)
+                    buffer_from_stream = self._wrapped_stream.read(current_max_buffer_size)
 
             if buffer_from_stream:
+                # update the buffer with new data from the wrapped stream
+                # we need to note down the start position and size of the buffer, in case seek is performed later
                 self._buffer = BytesIO(buffer_from_stream)
+                self._current_buffer_start = self._position
+                self._current_buffer_size = len(buffer_from_stream)
+
+                # read the remaining bytes from the new buffer and update position
                 second_read_buffer = self._buffer.read(bytes_remaining)
-                bytes_read += len(second_read_buffer)
                 read_buffer += second_read_buffer
+                self._position += len(second_read_buffer)
 
-        self._position += bytes_read
         return read_buffer
 
     def readable(self):
@@ -437,6 +456,15 @@ def seek(self, offset, whence=0):
         elif pos < 0:
             pos = 0
 
+        # check if buffer is still valid
+        # if not, drop buffer
+        if pos < self._current_buffer_start or pos >= self._current_buffer_start + self._current_buffer_size:
+            self._buffer.close()
+            self._buffer = BytesIO()
+        else:  # if yes seek to correct position
+            delta = pos - self._current_buffer_start
+            self._buffer.seek(delta, SEEK_SET)
+
         self._position = pos
         return pos
 

azure/multiapi/storage/v2017_04_17/blob/blockblobservice.py

@@ -348,7 +348,9 @@ def create_blob_from_path(
             that was sent. This is primarily valuable for detecting bitflips on
             the wire if using http instead of https as https (the default) will
             already validate. Note that this MD5 hash is not stored with the
-            blob.
+            blob. Also note that if enabled, the memory-efficient upload algorithm
+            will not be used, because computing the MD5 hash requires buffering
+            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
         :param progress_callback:
             Callback for progress with signature function(current, total) where
             current is the number of bytes transfered so far, and total is the
@@ -441,7 +443,9 @@ def create_blob_from_stream(
             that was sent. This is primarily valuable for detecting bitflips on
             the wire if using http instead of https as https (the default) will
             already validate. Note that this MD5 hash is not stored with the
-            blob.
+            blob. Also note that if enabled, the memory-efficient upload algorithm
+            will not be used, because computing the MD5 hash requires buffering
+            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
         :param progress_callback:
             Callback for progress with signature function(current, total) where
             current is the number of bytes transfered so far, and total is the
@@ -507,6 +511,7 @@ def create_blob_from_stream(
         if (self.key_encryption_key is not None) and (adjusted_count is not None):
             adjusted_count += (16 - (count % 16))
 
+        # Do single put if the size is smaller than MAX_SINGLE_PUT_SIZE
         if adjusted_count is not None and (adjusted_count < self.MAX_SINGLE_PUT_SIZE):
             if progress_callback:
                 progress_callback(0, count)
@@ -530,10 +535,10 @@ def create_blob_from_stream(
                 progress_callback(count, count)
 
             return resp
-        else:
+        else:  # Size is larger than MAX_SINGLE_PUT_SIZE, must upload with multiple put_block calls
             cek, iv, encryption_data = None, None, None
 
-            use_original_upload_path = use_byte_buffer or self.require_encryption or \
+            use_original_upload_path = use_byte_buffer or validate_content or self.require_encryption or \
                                        self.MAX_BLOCK_SIZE < self.MIN_LARGE_BLOCK_UPLOAD_THRESHOLD or \
                                        hasattr(stream, 'seekable') and not stream.seekable() or \
                                        not hasattr(stream, 'seek') or not hasattr(stream, 'tell')

azure/multiapi/storage/v2017_04_17/blob/models.py

@@ -115,6 +115,18 @@ class BlobProperties(object):
         Stores all the content settings for the blob.
     :ivar ~azure.storage.blob.models.LeaseProperties lease:
         Stores all the lease information for the blob.
+    :ivar StandardBlobTier blob_tier:
+        Indicates the access tier of the blob. The hot tier is optimized
+        for storing data that is accessed frequently. The cool storage tier
+        is optimized for storing data that is infrequently accessed and stored
+        for at least a month. The archive tier is optimized for storing
+        data that is rarely accessed and stored for at least six months
+        with flexible latency requirements.
+    :ivar datetime blob_tier_change_time:
+        Indicates when the access tier was last changed.
+    :ivar bool blob_tier_inferred:
+        Indicates whether the access tier was inferred by the service.
+        If false, it indicates that the tier was set explicitly.
     '''
 
     def __init__(self):
@@ -129,6 +141,9 @@ def __init__(self):
         self.copy = CopyProperties()
         self.content_settings = ContentSettings()
         self.lease = LeaseProperties()
+        self.blob_tier = None
+        self.blob_tier_change_time = None
+        self.blob_tier_inferred = False
 
 
 class ContentSettings(object):

azure/multiapi/storage/v2017_04_17/common/_constants.py

@@ -15,7 +15,7 @@
 import platform
 
 __author__ = 'Microsoft Corp. <[email protected]>'
-__version__ = '0.37.0'
+__version__ = '0.37.1'
 
 # UserAgent string sample: 'Azure-Storage/0.37.0-0.38.0 (Python CPython 3.4.2; Windows 8)'
 # First version(0.37.0) is the common package, and the second version(0.38.0) is the service package

azure/multiapi/storage/v2017_04_17/common/_deserialization.py

@@ -65,6 +65,7 @@ def _get_download_size(start_range, end_range, resource_size):
     'x-ms-blob-sequence-number': (None, 'page_blob_sequence_number', _int_to_str),
     'x-ms-blob-committed-block-count': (None, 'append_blob_committed_block_count', _int_to_str),
     'x-ms-access-tier': (None, 'blob_tier', _to_str),
+    'x-ms-access-tier-change-time': (None, 'blob_tier_change_time', parser.parse),
     'x-ms-access-tier-inferred': (None, 'blob_tier_inferred', _bool),
     'x-ms-archive-status': (None, 'rehydration_status', _to_str),
     'x-ms-share-quota': (None, 'quota', _int_to_str),

azure/multiapi/storage/v2017_04_17/common/_encryption.py

@@ -214,7 +214,7 @@ def _validate_and_unwrap_cek(encryption_data, key_encryption_key=None, key_resol
         instance variables for more details.
     :param func key_resolver:
         A function used that, given a key_id, will return a key_encryption_key. Please refer 
-        to high service object instance variables for more details.
+        to high-level service object instance variables for more details.
     :return: the content_encryption_key stored in the encryption_data object.
     :rtype: bytes[]
     '''

azure/multiapi/storage/v2017_04_17/common/models.py

@@ -124,14 +124,18 @@ class RetryContext(object):
         The request sent to the storage service.
     :ivar ~azure.storage.common._http.HTTPResponse response:
         The response returned by the storage service.
-    :ivar LocationMode location_mode: 
+    :ivar LocationMode location_mode:
         The location the request was sent to.
+    :ivar Exception exception:
+        The exception that just occurred. The type could either be AzureException (for HTTP errors),
+        or other Exception types from lower layers, which are kept unwrapped for easier processing.
     '''
 
     def __init__(self):
         self.request = None
         self.response = None
         self.location_mode = None
+        self.exception = None
 
 
 class LocationMode(object):
@@ -593,10 +597,10 @@ def __init__(self, read=False, write=False, delete=False, list=False,
         self.process = process or ('p' in _str)
 
     def __or__(self, other):
-        return ResourceTypes(_str=str(self) + str(other))
+        return AccountPermissions(_str=str(self) + str(other))
 
     def __add__(self, other):
-        return ResourceTypes(_str=str(self) + str(other))
+        return AccountPermissions(_str=str(self) + str(other))
 
     def __str__(self):
         return (('r' if self.read else '') +