White list the module to run through style check

scripts/ci/test_static.sh

@@ -1,15 +1,13 @@
 #!/usr/bin/env bash
 
-set -e
-
 . $(cd $(dirname $0); pwd)/artifacts.sh
 
 ls -la $share_folder/build
 
 ALL_MODULES=`find $share_folder/build/ -name "*.whl"`
 
 [ -d privates ] && pip install privates/*.whl
-pip install pylint flake8
+pip install pylint
 pip install $ALL_MODULES
 
 echo '=== List installed packages'
@@ -20,5 +18,60 @@ echo '=== Begin testing'
 proc_number=`python -c 'import multiprocessing; print(multiprocessing.cpu_count())'`
 
 echo "Run pylint with $proc_number proc."
-pylint azure.cli --rcfile=./pylintrc -j $proc_number
 
+# Uncommented after all conversion is done
+# pylint azure.cli --rcfile=./pylintrc -j $proc_number
+
+proc_number=`python -c 'import multiprocessing; print(multiprocessing.cpu_count())'`
+exit_code=0
+
+run_style() {
+    pylint $1 --rcfile=./pylintrc -j $proc_number
+    let exit_code=$exit_code+$?
+}
+
+
+set +e
+
+run_style azure.cli.core
+#run_style azure.cli.command_modules.acr
+#run_style azure.cli.command_modules.acs
+#run_style azure.cli.command_modules.advisor
+#run_style azure.cli.command_modules.appservice
+#run_style azure.cli.command_modules.backup
+#run_style azure.cli.command_modules.batch
+#run_style azure.cli.command_modules.batchai
+run_style azure.cli.command_modules.billing
+run_style azure.cli.command_modules.cdn
+run_style azure.cli.command_modules.cloud
+#run_style azure.cli.command_modules.cognitiveservices
+run_style azure.cli.command_modules.configure
+#run_style azure.cli.command_modules.consumption
+#run_style azure.cli.command_modules.container
+#run_style azure.cli.command_modules.cosmosdb
+#run_style azure.cli.command_modules.dla
+#run_style azure.cli.command_modules.dls
+#run_style azure.cli.command_modules.eventgrid
+run_style azure.cli.command_modules.extension
+run_style azure.cli.command_modules.feedback
+run_style azure.cli.command_modules.find
+run_style azure.cli.command_modules.interactive
+#run_style azure.cli.command_modules.iot
+#run_style azure.cli.command_modules.keyvault
+#run_style azure.cli.command_modules.lab
+#run_style azure.cli.command_modules.monitor
+run_style azure.cli.command_modules.network
+#run_style azure.cli.command_modules.nspkg
+#run_style azure.cli.command_modules.profile
+#run_style azure.cli.command_modules.rdbms
+run_style azure.cli.command_modules.redis
+#run_style azure.cli.command_modules.reservations
+run_style azure.cli.command_modules.resource
+run_style azure.cli.command_modules.role
+#run_style azure.cli.command_modules.servicefabric
+#run_style azure.cli.command_modules.sql
+#run_style azure.cli.command_modules.storage
+#run_style azure.cli.command_modules.testsdk
+#run_style azure.cli.command_modules.vm
+
+exit $exit_code

tools/.vscode/settings.json

@@ -0,0 +1,2 @@
+{
+}

tools/automation/__main__.py

@@ -7,13 +7,16 @@
 import sys
 import automation.verify
 import automation.clibuild
+import automation.style
+
 
 def main():
     parser = argparse.ArgumentParser(prog='Azure CLI build tools')
 
     sub_parser = parser.add_subparsers(title='azure cli tools sub commands')
     automation.verify.init_args(sub_parser)
     automation.clibuild.init_args(sub_parser)
+    automation.style.init_args(sub_parser)
 
     if sys.argv[1:]:
         args = parser.parse_args()
@@ -24,4 +27,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-

tools/automation/clibuild/__init__.py

@@ -20,19 +20,19 @@
 
 
 def build_debian(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
-    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION='+cli_version, '-e', 'BUILD_ARTIFACT_DIR=/artifacts',
-           '-v', artifact_dir+':/artifacts', 'ubuntu:14.04', '/bin/bash', '-cx',
-           'apt-get update && apt-get install -y git wget && ' \
-           'git clone --progress --verbose {} --branch {} /repo_clone && cd /repo_clone ' \
-           '&& build_scripts/debian/build.sh /repo_clone'.format(git_url, git_branch)]
+    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version, '-e', 'BUILD_ARTIFACT_DIR=/artifacts',
+           '-v', artifact_dir + ':/artifacts', 'ubuntu:14.04', '/bin/bash', '-cx',
+           'apt-get update && apt-get install -y git wget && git clone --progress --verbose {} --branch {} /repo_clone '
+           '&& cd /repo_clone && build_scripts/debian/build.sh /repo_clone'.format(git_url, git_branch)]
     container_id = check_output(cmd, universal_newlines=True).strip()
     print('Debian build running. Use `docker logs -f {}`'.format(container_id))
     exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
     print('COMPLETED Debian build. Exit code {}'.format(exit_code))
 
 
 def build_docker(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
-    cmd = ['docker', 'build', '--no-cache', '--quiet', '--build-arg', 'BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`"', '--build-arg', 'CLI_VERSION='+cli_version, get_repo_root()]
+    cmd = ['docker', 'build', '--no-cache', '--quiet', '--build-arg', 'BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`"',
+           '--build-arg', 'CLI_VERSION=' + cli_version, get_repo_root()]
     print('Docker build started.')
     image_id = check_output(cmd, universal_newlines=True).strip()
     image_id = image_id.split(':')[1]
@@ -43,24 +43,22 @@ def build_docker(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
 
 
 def build_rpm(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
-    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION='+cli_version, '-e', 'REPO_PATH=/repo_clone',
-           '-v', artifact_dir+':/artifacts', 'centos:7', '/bin/bash', '-cx',
-           'yum check-update; yum install -y gcc git rpm-build rpm-devel rpmlint make bash coreutils diffutils ' \
-           'patch rpmdevtools python libffi-devel python-devel openssl-devel wget && ' \
-           'git clone --progress --verbose {} --branch {} /repo_clone && cd /repo_clone ' \
-           '&& rpmbuild -v -bb --clean build_scripts/rpm/azure-cli.spec ' \
-           '&& cp /root/rpmbuild/RPMS/x86_64/* /artifacts/'.format(git_url, git_branch)]
+    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version, '-e', 'REPO_PATH=/repo_clone',
+           '-v', artifact_dir + ':/artifacts', 'centos:7', '/bin/bash', '-cx',
+           'yum check-update; yum install -y gcc git rpm-build rpm-devel rpmlint make bash coreutils diffutils patch '
+           'rpmdevtools python libffi-devel python-devel openssl-devel wget && git clone --progress --verbose {} '
+           '--branch {} /repo_clone && cd /repo_clone && rpmbuild -v -bb --clean build_scripts/rpm/azure-cli.spec && '
+           'cp /root/rpmbuild/RPMS/x86_64/* /artifacts/'.format(git_url, git_branch)]
     container_id = check_output(cmd, universal_newlines=True).strip()
     print('RPM build running. Use `docker logs -f {}`'.format(container_id))
     exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
     print('COMPLETED RPM build. Exit code {}'.format(exit_code))
 
 
 def build_pypi(git_url, git_branch, _, artifact_dir, arg_ns=None):
-    cmd = ['docker', 'run', '-d', '-v', artifact_dir+':/artifacts', 'python:3.6', '/bin/bash', '-cx',
-           'mkdir /artifacts/pypi ' \
-           '&& git clone --progress --verbose {} --branch {} /repo_clone && cd /repo_clone ' \
-           '&& python build_scripts/pypi/build.py /artifacts/pypi /repo_clone'.format(git_url, git_branch)]
+    cmd = ['docker', 'run', '-d', '-v', artifact_dir + ':/artifacts', 'python:3.6', '/bin/bash', '-cx',
+           'mkdir /artifacts/pypi && git clone --progress --verbose {} --branch {} /repo_clone && cd /repo_clone && '
+           'python build_scripts/pypi/build.py /artifacts/pypi /repo_clone'.format(git_url, git_branch)]
     container_id = check_output(cmd, universal_newlines=True).strip()
     print('Python pypi build message: The version numbers of packages will be as defined in source code.')
     print('Python pypi build running. Use `docker logs -f {}`'.format(container_id))
@@ -69,7 +67,7 @@ def build_pypi(git_url, git_branch, _, artifact_dir, arg_ns=None):
 
 
 def build_msi(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
-    #TODO
+    # TODO
     print('SKIPPED MSI build. Not Yet Implemented. Please build manually.')
 
 
@@ -78,13 +76,16 @@ def build_homebrew(git_url, git_branch, cli_version, artifact_dir, arg_ns=None):
         print('Homebrew message : The Homebrew formula requires CLI packages to be available on public PyPI. '
               'Version {} of the CLI does not appear to be on PyPI. '
               'If it was just updated, this message can be safely ignored.'.format(cli_version))
-    upstream_url = arg_ns.homebrew_upstream_url or 'https://github.com/Azure/azure-cli/archive/azure-cli-{cli_version}.tar.gz'.format(cli_version=cli_version)
-    print('Homebrew message: The generated formula uses the latest public packages that are available on PyPI, not the code in your Git repo.')
-    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION='+cli_version, '-e', 'BUILD_ARTIFACT_DIR=/artifacts',
-           '-e', 'UPSTREAM_URL='+upstream_url,
-           '-v', artifact_dir+':/artifacts', 'python:3.6', '/bin/bash', '-cx',
-           'pip install sh && git clone --progress --verbose {} --branch {} /repo_clone && cd /repo_clone ' \
-           '&& python build_scripts/homebrew/formula-generate.py'.format(git_url, git_branch)]
+
+    upstream_url = arg_ns.homebrew_upstream_url or 'https://github.com/Azure/azure-cli/archive/azure-cli-{cli_version}.tar.gz'.format(
+        cli_version=cli_version)
+    print('Homebrew message: The generated formula uses the latest public packages that are available on PyPI, '
+          'not the code in your Git repo.')
+    cmd = ['docker', 'run', '-d', '-e', 'CLI_VERSION=' + cli_version, '-e', 'BUILD_ARTIFACT_DIR=/artifacts',
+           '-e', 'UPSTREAM_URL=' + upstream_url,
+           '-v', artifact_dir + ':/artifacts', 'python:3.6', '/bin/bash', '-cx',
+           'pip install sh && git clone --progress --verbose {} --branch {} /repo_clone && cd /repo_clone && '
+           'python build_scripts/homebrew/formula-generate.py'.format(git_url, git_branch)]
     container_id = check_output(cmd, universal_newlines=True).strip()
     print('Homebrew formula generation running. Use `docker logs -f {}`'.format(container_id))
     exit_code = check_output(['docker', 'wait', container_id], universal_newlines=True).strip()
@@ -112,15 +113,18 @@ def cli_build(args):
     git_url = args.git_clone_url
     git_branch = args.git_clone_branch
     cli_version = args.cli_version
-    artifact_dir = tempfile.mkdtemp(prefix='cli-build-{}-'.format(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')), dir=os.getcwd())
+    artifact_dir = tempfile.mkdtemp(
+        prefix='cli-build-{}-'.format(datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')), dir=os.getcwd())
     if len(build_types) == 1 and build_types[0] == '*':
         build_types = BUILD_TYPES
     print_heading('Building for {} from branch {} of {} '
                   'and version number will be {}\n'
-                  'Build artifacts will be in {}'.format(', '.join(build_types), git_branch, git_url, cli_version, artifact_dir))
+                  'Build artifacts will be in {}'.format(', '.join(build_types), git_branch, git_url, cli_version,
+                                                         artifact_dir))
     from concurrent.futures import ThreadPoolExecutor, as_completed
     with ThreadPoolExecutor(max_workers=len(build_types)) as executor:
-        tasks = {executor.submit(build_dispatch, bt, git_url, git_branch, cli_version, artifact_dir, arg_ns=args) for bt in build_types}
+        tasks = {executor.submit(build_dispatch, bt, git_url, git_branch, cli_version, artifact_dir, arg_ns=args) for bt
+                 in build_types}
         for t in as_completed(tasks):
             t.result()
     print('Done.')
@@ -130,9 +134,16 @@ def init_args(root):
     cli_build_parser = root.add_parser('build', help='Build the CLI. Docker is required.')
     cli_build_parser.set_defaults(func=cli_build)
     git_args = cli_build_parser.add_argument_group('Git Clone Arguments')
-    git_args.add_argument('-b', '--git-clone-branch', dest='git_clone_branch', help='Branch name that should be checked out. (default: %(default)s)', default='master')
-    git_args.add_argument('-u', '--git-clone-url', dest='git_clone_url', help='The url to clone. This will be passed to `git clone`. (default: %(default)s)', default='https://github.com/Azure/azure-cli.git')
-    cli_build_parser.add_argument('-t', '--type', dest='build_types', required=True, nargs='+', choices=BUILD_TYPES+['*'], help="Space separated list of the artifacts to build. Use '*' for all.")
-    cli_build_parser.add_argument('-c', '--cli-version', dest='cli_version', required=True, help="The version of the build. (ignored for 'pypi' type)")
+    git_args.add_argument('-b', '--git-clone-branch', dest='git_clone_branch',
+                          help='Branch name that should be checked out. (default: %(default)s)', default='master')
+    git_args.add_argument('-u', '--git-clone-url', dest='git_clone_url',
+                          help='The url to clone. This will be passed to `git clone`. (default: %(default)s)',
+                          default='https://github.com/Azure/azure-cli.git')
+    cli_build_parser.add_argument('-t', '--type', dest='build_types', required=True, nargs='+',
+                                  choices=BUILD_TYPES + ['*'],
+                                  help="Space separated list of the artifacts to build. Use '*' for all.")
+    cli_build_parser.add_argument('-c', '--cli-version', dest='cli_version', required=True,
+                                  help="The version of the build. (ignored for 'pypi' type)")
     homebrew_args = cli_build_parser.add_argument_group('Homebrew Specific Arguments')
-    homebrew_args.add_argument('--homebrew-upstream-url', dest='homebrew_upstream_url', help='The upstream URL to specify in the formula.')
+    homebrew_args.add_argument('--homebrew-upstream-url', dest='homebrew_upstream_url',
+                               help='The upstream URL to specify in the formula.')

tools/automation/commandlint/run.py

@@ -38,7 +38,7 @@ def dump_no_help(modules):
         try:
             import_module('azure.cli.command_modules.' + mod).load_params(mod)
         except Exception as ex:
-            print("EXCEPTION: " + str(mod))
+            print("EXCEPTION: {} for module {}".format(ex, str(mod)))
 
     _update_command_definitions(cmd_table)
     add_id_parameters(cmd_table)
@@ -66,8 +66,8 @@ def dump_no_help(modules):
         param_list = set()
         for key in cmd_table[cmd].arguments:
             name = cmd_table[cmd].arguments[key].name
-            if not cmd_table[cmd].arguments[key].type.settings.get('help') and \
-                    name not in white_list_parameters.get(cmd, []):
+            if not cmd_table[cmd].arguments[key].type.settings.get('help') and name not in white_list_parameters.get(
+                    cmd, []):
                 exit_val = 1
                 param_list.add(name)
         if param_list:

tools/automation/style/__init__.py

@@ -2,3 +2,106 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
+
+
+import argparse
+import os
+import multiprocessing
+import sys
+from subprocess import call
+
+import automation.utilities.path as automation_path
+
+
+def run_pylint(modules):
+    print('\n\nRun pylint')
+    print('Modules: {}'.format(', '.join([name for name, _ in modules if not name.endswith('-nspkg')])))
+
+    modules_list = ' '.join(
+        [os.path.join(path, 'azure') for name, path in modules if not name.endswith('-nspkg')])
+    arguments = '{} --rcfile={} -j {}'.format(
+        modules_list,
+        os.path.join(automation_path.get_repo_root(), 'pylintrc'),
+        multiprocessing.cpu_count())
+
+    return_code = call(('python -m pylint ' + arguments).split())
+
+    if return_code:
+        print('Pylint failed')
+    else:
+        print('Pylint passed')
+
+    return return_code
+
+
+def run_pep8(modules):
+    print('\n\nRun flake8 for PEP8 compliance')
+    print('Modules: {}'.format(', '.join(name for name, _ in modules)))
+
+    command = 'flake8 --statistics --exclude=azure_bdist_wheel.py --append-config={} {}'.format(
+        os.path.join(automation_path.get_repo_root(), '.flake8'),
+        ' '.join(path for _, path in modules))
+
+    return_code = call(command.split())
+    if return_code:
+        print('Flake8 failed')
+    else:
+        print('Flake8 passed')
+
+    return return_code
+
+
+def define_arguments(parser):
+    parser.add_argument('--ci', action='store_true', help='Run in CI mode')
+    parser.add_argument('--pep8', dest='suites', action='append_const', const='pep8',
+                        help='Run flake8 to check PEP8')
+    parser.add_argument('--pylint', dest='suites', action='append_const', const='pylint',
+                        help='Run pylint')
+    parser.add_argument('--module', dest='modules', action='append',
+                        help='The modules on which the style check should run. Accept short names, '
+                             'except azure-cli, azure-cli-core and azure-cli-nspkg')
+
+
+def main_style(args):
+    if args.ci:
+        # When the command is run in CI mode all the other parameters are ignored
+        selected_modules = automation_path.filter_user_selected_modules(None)
+
+        # Run pylint on all modules
+        return_code_sum = run_pylint(selected_modules)
+
+        # Run flake8 on modules
+        return_code_sum += run_pep8(selected_modules)
+
+        sys.exit(return_code_sum)
+
+    selected_modules = automation_path.filter_user_selected_modules(args.modules)
+    if not selected_modules:
+        sys.stderr('No module is selected.\n')
+        sys.exit(1)
+
+    if not args.suites:
+        return_code_sum = run_pylint(selected_modules) + run_pep8(selected_modules)
+    else:
+        return_code_sum = 0
+        if 'pep8' in args.suites:
+            return_code_sum += run_pep8(selected_modules)
+
+        if 'pylint' in args.suites:
+            return_code_sum += run_pylint(selected_modules)
+
+    sys.exit(return_code_sum)
+
+
+def init_args(root):
+    parser = root.add_parser('style', help="Code style check.")
+    define_arguments(parser)
+    parser.set_defaults(func=main_style)
+
+
+def legacy_entry():
+    sys.stderr.write("The check_style command is going to be replaced by 'azdev style' command.\n\n")
+    parser = argparse.ArgumentParser('Code style check.')
+    define_arguments(parser)
+    args = parser.parse_args()
+    main_style(args)

tools/automation/style/pylint_disable_check.py

@@ -59,7 +59,6 @@ def main():
     src_folder = os.path.join(get_repo_root(), 'src')
     all_rules = [e for e in get_all_rules(src_folder)]
 
-
     with open('pylint_report.txt', 'w') as f:
         f.write('GROUP BY RULES\n')
         f.writelines(tabulate.tabulate(sorted(group_by_rules(all_rules), key=lambda each: each[1], reverse=True),
@@ -75,4 +74,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()