[auto release] support for typespec (Azure#35137)

* support for typespec

* fix whl package install

* fix test

Author: msyyc

scripts/auto_release/PythonSdkLiveTest.yml

@@ -79,12 +79,8 @@ jobs:
         # import env variable
         export AZURE_TEST_RUN_LIVE=$(AZURE_TEST_RUN_LIVE)
         export TENANT_ID=$(ENV-TENANT-ID)
-        export CLIENT_ID=$(ENV-CLIENT-ID)
-        export CLIENT_SECRET=$(ENV-CLIENT-SECRET)
         export SUBSCRIPTION_ID=$(ENV-SUBSCRIPTION-ID)
         export AZURE_TENANT_ID=$(ENV-TENANT-ID)
-        export AZURE_CLIENT_ID=$(ENV-CLIENT-ID)
-        export AZURE_CLIENT_SECRET=$(ENV-CLIENT-SECRET)
         export AZURE_SUBSCRIPTION_ID=$(ENV-SUBSCRIPTION-ID)
         export ISSUE_LINK=$(ISSUE_LINK)
         export SCRIPT_PATH=$script_path

scripts/auto_release/main.py

@@ -13,6 +13,7 @@
 from github import Github
 from azure.storage.blob import BlobServiceClient, ContainerClient
 from datetime import datetime, timedelta
+import importlib
 
 _LOG = logging.getLogger()
 
@@ -98,22 +99,6 @@ def current_time_month() -> str:
     date = time.localtime(time.time())
     return '{}-{:02d}'.format(date.tm_year, date.tm_mon)
 
-
-def set_test_env_var():
-    setting_path = str(Path(os.getenv('SCRIPT_PATH')) / 'mgmt_settings_real_.py')
-    # edit mgmt_settings_real.py
-    with open(setting_path, 'r') as file_in:
-        list_in = file_in.readlines()
-
-    for i in range(0, len(list_in)):
-        list_in[i] = list_in[i].replace('ENV_TENANT_ID', os.environ['TENANT_ID'])
-        list_in[i] = list_in[i].replace('ENV_CLIENT_ID', os.environ['CLIENT_ID'])
-        list_in[i] = list_in[i].replace('ENV_CLIENT_SECRET', os.environ['CLIENT_SECRET'])
-        list_in[i] = list_in[i].replace('ENV_SUBSCRIPTION_ID', os.environ['SUBSCRIPTION_ID'])
-    with open(str(Path('tools/azure-sdk-tools/devtools_testutils/mgmt_settings_real.py')), 'w') as file_out:
-        file_out.writelines(list_in)
-
-
 class CodegenTestPR:
     """
     This class can generate SDK code, run live test and create RP
@@ -131,6 +116,7 @@ def __init__(self):
         self.test_folder = os.getenv('TEST_FOLDER', '')
 
         self.package_name = '' # 'dns' of 'sdk/compute/azure-mgmt-dns'
+        self.whole_package_name = '' # 'azure-mgmt-dns'
         self.new_branch = ''
         self.sdk_folder = ''  # 'compute' of 'sdk/compute/azure-mgmt-dns'
         self.autorest_result = ''
@@ -183,16 +169,21 @@ def checkout_azure_default_branch(self):
         if self.spec_repo:
             os.chdir(Path(self.spec_repo))
             self.checkout_branch("DEBUG_REST_BRANCH", "azure-rest-api-specs")
+    
+    @property
+    def from_swagger(self) -> bool:
+        return "readme.md" in self.spec_readme
 
     def generate_code(self):
         self.checkout_azure_default_branch()
 
         # prepare input data
+        file_name = "relatedReadmeMdFiles" if self.from_swagger else "relatedTypeSpecProjectFolder"
         input_data = {
             'headSha': self.get_latest_commit_in_swagger_repo(),
             'repoHttpsUrl': "https://github.com/Azure/azure-rest-api-specs",
             'specFolder': self.spec_repo,
-            'relatedReadmeMdFiles': [self.readme_local_folder()]
+            file_name: [self.readme_local_folder()],
         }
         log(str(input_data))
 
@@ -216,7 +207,8 @@ def generate_code(self):
 
     def get_package_name_with_autorest_result(self):
         generate_result = self.get_autorest_result()
-        self.package_name = generate_result["packages"][0]["packageName"].split('-', 2)[-1]
+        self.whole_package_name = generate_result["packages"][0]["packageName"]
+        self.package_name = self.whole_package_name.split('-', 2)[-1]
 
     def prepare_branch_with_readme(self):
         self.generate_code()
@@ -229,7 +221,7 @@ def create_new_branch(self):
         print_check(f'git checkout -b {self.new_branch}')
 
     def check_sdk_readme(self):
-        sdk_readme = str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}/README.md'))
+        sdk_readme = str(Path(f'sdk/{self.sdk_folder}/{self.whole_package_name}/README.md'))
 
         def edit_sdk_readme(content: List[str]):
             for i in range(0, len(content)):
@@ -249,20 +241,20 @@ def readme_python_md_path(self)-> Path:
     # Use the template to update readme and setup by packaging_tools
     @return_origin_path
     def check_file_with_packaging_tool(self):
-        python_md = self.readme_python_md_path
+        print_check(f"pip install {self.get_whl_package}")
+        module = importlib.import_module(self.whole_package_name.replace("-", "."))
         title = ""
-        if python_md.exists():
-            with open(python_md, "r") as file_in:
-                md_content = file_in.readlines()
-            for line in md_content:
-                if "title:" in line:
-                    title = line.replace("title:", "").strip(" \r\n")
-                    break
-        else:
-            log(f"{python_md} does not exist")
+        for item in getattr(module, "__all__"):
+            if item.endswith("Client"):
+                title = item
+                break
+        
+        if not title:
+            log(f"Can not find the title in {self.whole_package_name}")
+        
         os.chdir(Path(f'sdk/{self.sdk_folder}'))
         # add `title` and update `is_stable` in sdk_packaging.toml
-        toml = Path(f"azure-mgmt-{self.package_name}") / "sdk_packaging.toml"
+        toml = Path(self.whole_package_name) / "sdk_packaging.toml"
         stable_config = "is_stable = " + ("true" if self.tag_is_stable else "false") + "\n"
         if toml.exists():
             def edit_toml(content: List[str]):
@@ -282,7 +274,7 @@ def edit_toml(content: List[str]):
         else:
             log(f"{os.getcwd()}/{toml} does not exist")
 
-        print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}')
+        print_check(f'python -m packaging_tools --build-conf {self.whole_package_name}')
         log('packaging_tools --build-conf successfully ')
 
     def check_pprint_name(self):
@@ -390,7 +382,7 @@ def check_changelog_file(self):
             self.edit_changelog()
 
     def check_dev_requirement(self):
-        file = Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}/dev_requirements.txt')
+        file = Path(f'sdk/{self.sdk_folder}/{self.whole_package_name}/dev_requirements.txt')
         content = [
             "-e ../../../tools/azure-sdk-tools\n",
             "../../identity/azure-identity\n"
@@ -407,22 +399,23 @@ def check_package_size(self):
                     self.check_package_size_result.append(f'ERROR: Package size is over 2MBytes: {Path(package).name}!!!')
 
     def check_model_flatten(self):
-        last_version = self.get_last_release_version()
-        if last_version == "" or last_version.startswith("1.0.0b"):
-            with open(self.readme_md_path, 'r') as file_in:
-                readme_md_content = file_in.read()
-
-            with open(self.readme_python_md_path, 'r') as file_in:
-                readme_python_md_content = file_in.read()
-            
-            if "flatten-models: false" not in readme_md_content and "flatten-models: false" not in readme_python_md_content and self.issue_link:
-                api = Github(self.bot_token).get_repo("Azure/sdk-release-request")
-                issue_number = int(self.issue_link.split('/')[-1])
-                issue = api.get_issue(issue_number)
-                assignee = issue.assignee.login if issue.assignee else ""
-                message = "please set `flatten-models: false` in readme.md or readme.python.md"
-                issue.create_comment(f'@{assignee}, {message}')
-                raise Exception(message)
+        if self.from_swagger:
+            last_version = self.get_last_release_version()
+            if last_version == "" or last_version.startswith("1.0.0b"):
+                with open(self.readme_md_path, 'r') as file_in:
+                    readme_md_content = file_in.read()
+
+                with open(self.readme_python_md_path, 'r') as file_in:
+                    readme_python_md_content = file_in.read()
+                
+                if "flatten-models: false" not in readme_md_content and "flatten-models: false" not in readme_python_md_content and self.issue_link:
+                    api = Github(self.bot_token).get_repo("Azure/sdk-release-request")
+                    issue_number = int(self.issue_link.split('/')[-1])
+                    issue = api.get_issue(issue_number)
+                    assignee = issue.assignee.login if issue.assignee else ""
+                    message = "please set `flatten-models: false` in readme.md or readme.python.md"
+                    issue.create_comment(f'@{assignee}, {message}')
+                    raise Exception(message)
 
     def check_file(self):
         self.check_file_with_packaging_tool()
@@ -435,7 +428,7 @@ def check_file(self):
         self.check_model_flatten()
 
     def sdk_code_path(self) -> str:
-        return str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}'))
+        return str(Path(f'sdk/{self.sdk_folder}/{self.whole_package_name}'))
 
     @property
     def is_single_path(self) -> bool:
@@ -451,7 +444,6 @@ def install_package_locally(self):
 
     def prepare_test_env(self):
         self.install_package_locally()
-        set_test_env_var()
 
     @staticmethod
     def is_live_test()-> bool:
@@ -512,7 +504,7 @@ def create_pr_proc(self):
         pr_body = "" if not self.check_package_size_result else "{}\n".format("\n".join(self.check_package_size_result))
         pr_body = pr_body + "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link)
         if not self.is_single_path:
-            pr_body += f'\nBuildTargetingString\n  azure-mgmt-{self.package_name}\nSkip.CreateApiReview\ntrue'
+            pr_body += f'\nBuildTargetingString\n  {self.whole_package_name}\nSkip.CreateApiReview\ntrue'
         res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body)
 
         # Add issue link on PR
@@ -547,6 +539,10 @@ def get_private_package(self) -> List[str]:
         content = self.get_autorest_result()
         return content["packages"][0]["artifacts"]
 
+    @property
+    def get_whl_package(self) -> str:
+        return [package for package in self.get_private_package() if package.endswith('.whl')][0]
+
     def upload_private_package_proc(self, container_name: str):
         container_client = ContainerClient.from_connection_string(conn_str=self.conn_str, container_name=container_name)
         private_package = self.get_private_package()