Add blob prefix when synchronizing extensions (Azure#3346)

Author: fengzhou-msft

scripts/ci/build_ext_cmd_tree.sh

@@ -14,9 +14,8 @@ az extension list-available -otable
 # turn off telemetry as it crowds output
 export AZURE_CORE_COLLECT_TELEMETRY=False
 
-# wait for the index.json to be synced in storage account
-# Remove this when we can support using customized index.json
-sleep 360
+# use index.json in master branch
+export AZURE_EXTENSION_INDEX_URL=https://raw.githubusercontent.com/Azure/azure-cli-extensions/master/src/index.json
 
 output=$(az extension list-available --query [].name -otsv)
 # azure-cli-iot-ext is the deprecated old versions of the renamed azure-iot extension
@@ -32,7 +31,7 @@ for ext in $output; do
     fi
     filter_exts="${filter_exts} ${ext}"
     echo "Adding extension:" $ext
-    az extension add -n $ext
+    az extension add --upgrade -n $ext
     if [ $? != 0 ]
     then
         echo "Failed to load:" $ext

scripts/ci/sync_extensions.py

@@ -15,14 +15,15 @@
 STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_ACCOUNT')
 STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_TARGET_STORAGE_CONTAINER')
 COMMIT_NUM = os.getenv('AZURE_EXTENSION_COMMIT_NUM') or 1
+BLOB_PREFIX = os.getenv('AZURE_EXTENSION_BLOB_PREFIX')
 
 
 def _get_updated_extension_filenames():
     cmd = 'git --no-pager diff --diff-filter=ACMRT HEAD~{} -- src/index.json'.format(COMMIT_NUM)
     updated_content = check_output(cmd.split()).decode('utf-8')
     FILENAME_REGEX = r'"filename":\s+"(.*?)"'
-    added_ext_filenames = [re.findall(FILENAME_REGEX, line)[0] for line in updated_content.splitlines() if line.startswith('+') and not line.startswith('+++') and 'filename' in line]
-    deleted_ext_filenames = [re.findall(FILENAME_REGEX, line)[0] for line in updated_content.splitlines() if line.startswith('-') and not line.startswith('---') and 'filename' in line]
+    added_ext_filenames = {re.findall(FILENAME_REGEX, line)[0] for line in updated_content.splitlines() if line.startswith('+') and not line.startswith('+++') and 'filename' in line}
+    deleted_ext_filenames = {re.findall(FILENAME_REGEX, line)[0] for line in updated_content.splitlines() if line.startswith('-') and not line.startswith('---') and 'filename' in line}
     return added_ext_filenames, deleted_ext_filenames
 
 
@@ -53,19 +54,20 @@ def _sync_wheel(ext, updated_indexes, failed_urls, client, overwrite, temp_dir):
     download_url = ext['downloadUrl']
     whl_file = download_url.split('/')[-1]
     whl_path = os.path.join(temp_dir, whl_file)
+    blob_name = f'{BLOB_PREFIX}/{whl_file}' if BLOB_PREFIX else whl_file
     try:
         download_file(download_url, whl_path)
     except Exception:
         failed_urls.append(download_url)
         return
     if not overwrite:
-        exists = client.exists(container_name=STORAGE_CONTAINER, blob_name=whl_file)
+        exists = client.exists(container_name=STORAGE_CONTAINER, blob_name=blob_name)
         if exists:
             print("Skipping '{}' as it already exists...".format(whl_file))
             return
-    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=whl_file,
+    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=blob_name,
                                  file_path=os.path.abspath(whl_path))
-    url = client.make_blob_url(container_name=STORAGE_CONTAINER, blob_name=whl_file)
+    url = client.make_blob_url(container_name=STORAGE_CONTAINER, blob_name=blob_name)
     updated_index = ext
     updated_index['downloadUrl'] = url
     updated_indexes.append(updated_index)
@@ -104,12 +106,17 @@ def main():
     import tempfile
     from azure.storage.blob import BlockBlobService
 
-    added_ext_filenames = []
-    deleted_ext_filenames = []
+    net_added_ext_filenames = []
+    net_deleted_ext_filenames = []
     sync_all = (os.getenv('AZURE_SYNC_ALL_EXTENSIONS') and os.getenv('AZURE_SYNC_ALL_EXTENSIONS').lower() == 'true')
     if not sync_all:
         added_ext_filenames, deleted_ext_filenames = _get_updated_extension_filenames()
-        if not added_ext_filenames and not deleted_ext_filenames:
+        # when there are large amount of changes, for instance deleting a lot of old versions of extensions,
+        # git may not accurately recognize the right changes, so we need to compare added filenames and deleted filenames
+        # to get the real changed ones.
+        net_added_ext_filenames = added_ext_filenames - deleted_ext_filenames
+        net_deleted_ext_filenames = deleted_ext_filenames - added_ext_filenames
+        if not net_added_ext_filenames and not net_deleted_ext_filenames:
             print('index.json not changed. End task.')
             return
     temp_dir = tempfile.mkdtemp()
@@ -119,25 +126,33 @@ def main():
     target_index = DEFAULT_TARGET_INDEX_URL
     os.mkdir(os.path.join(temp_dir, 'target'))
     target_index_path = os.path.join(temp_dir, 'target', 'index.json')
-    download_file(target_index, target_index_path)
-
+    try:
+        download_file(target_index, target_index_path)
+    except Exception as ex:
+        if sync_all and '404' in str(ex):
+            initial_index = {"extensions": {}, "formatVersion": "1"}
+            open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True))
+        else:
+            raise
     client = BlockBlobService(account_name=STORAGE_ACCOUNT, account_key=STORAGE_ACCOUNT_KEY)
     updated_indexes = []
     failed_urls = []
     if sync_all:
         print('Syncing all extensions...\n')
         # backup the old index.json
-        client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name='index.json.sav',
+        backup_index_name = f'{BLOB_PREFIX}/index.json.sav' if BLOB_PREFIX else 'index.json.sav'
+        client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=backup_index_name,
                                      file_path=os.path.abspath(target_index_path))
-        inital_index = {"extensions": {}, "formatVersion": "1"}
-        open(target_index_path, 'w').write(json.dumps(inital_index, indent=4, sort_keys=True))
+        # start with an empty index.json to sync all extensions
+        initial_index = {"extensions": {}, "formatVersion": "1"}
+        open(target_index_path, 'w').write(json.dumps(initial_index, indent=4, sort_keys=True))
         for extension_name in current_extensions.keys():
             for ext in current_extensions[extension_name]:
                 print('Uploading {}'.format(ext['filename']))
                 _sync_wheel(ext, updated_indexes, failed_urls, client, True, temp_dir)
     else:
         NAME_REGEX = r'^(.*?)-\d+.\d+.\d+'
-        for filename in added_ext_filenames:
+        for filename in net_added_ext_filenames:
             extension_name = re.findall(NAME_REGEX, filename)[0].replace('_', '-')
             print('Uploading {}'.format(filename))
             ext = current_extensions[extension_name][-1]
@@ -147,17 +162,19 @@ def main():
                 _sync_wheel(ext, updated_indexes, failed_urls, client, True, temp_dir)
 
     print("")
-    _update_target_extension_index(updated_indexes, deleted_ext_filenames, target_index_path)
-    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name='index.json',
+    _update_target_extension_index(updated_indexes, net_deleted_ext_filenames, target_index_path)
+    index_name = f'{BLOB_PREFIX}/index.json' if BLOB_PREFIX else 'index.json'
+    client.create_blob_from_path(container_name=STORAGE_CONTAINER, blob_name=index_name,
                                  file_path=os.path.abspath(target_index_path))
+    print("\nSync finished.")
     if updated_indexes:
-        print("\nSync finished, extensions available in:")
+        print("New extensions available in:")
     for updated_index in updated_indexes:
         print(updated_index['downloadUrl'])
     shutil.rmtree(temp_dir)
 
     if failed_urls:
-        print("\nFailed to donwload and sync the following files. They are skipped:")
+        print("\nFailed to download and sync the following files. They are skipped:")
         for url in failed_urls:
             print(url)
         print("")

scripts/ci/update_ext_cmd_tree.py

@@ -16,6 +16,7 @@
 STORAGE_ACCOUNT_KEY = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_ACCOUNT_KEY')
 STORAGE_ACCOUNT = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_ACCOUNT')
 STORAGE_CONTAINER = os.getenv('AZURE_EXTENSION_CMD_TREE_STORAGE_CONTAINER')
+BLOB_PREFIX = os.getenv('AZURE_EXTENSION_CMD_TREE_BLOB_PREFIX')
 
 az_cli = get_default_cli()
 file_name = 'extCmdTreeToUpload.json'
@@ -75,6 +76,8 @@ def update_cmd_tree(ext_name):
 
 def upload_cmd_tree():
     blob_file_name = 'extensionCommandTree.json'
+    if BLOB_PREFIX:
+        blob_file_name = f'{BLOB_PREFIX}/{blob_file_name}'
     downloaded_file_name = 'extCmdTreeDownloaded.json'
     file_path = os.path.expanduser(os.path.join('~', '.azure', file_name))