[AIRFLOW-4364] Add Pylint to CI (apache#5238)

Author: BasPH

.pylintrc

@@ -67,3 +67,6 @@ apache-airflow-.*\+bin.tar.gz.*
 
 # vendored modules
 _vendor/*
+
+# Temporary list of files to make compatible with Pylint
+pylint_todo.txt

.rat-excludes

@@ -57,6 +57,11 @@ jobs:
       stage: pre-test
       install: pip install -e .[doc]
       script: docs/build.sh
+    - name: Pylint
+      stage: pre-test
+      install: pip install pylint~=2.3.1  # Ensure the same version as in setup.py
+      script: scripts/ci/ci_pylint.sh
+      cache: false
 cache:
   directories:
     - $HOME/.wheelhouse/

.travis.yml

@@ -275,6 +275,7 @@ meets these guidelines:
 1. If the pull request adds functionality, the docs should be updated as part of the same PR. Doc string are often sufficient.  Make sure to follow the Sphinx compatible standards.
 1. The pull request should work for Python 2.7 and 3.5. If you need help writing code that works in both Python 2 and 3, see the documentation at the [Python-Future project](http://python-future.org) (the future package is an Airflow requirement and should be used where possible).
 1. As Airflow grows as a project, we try to enforce a more consistent style and try to follow the Python community guidelines. We currently enforce most [PEP8](https://www.python.org/dev/peps/pep-0008/) and a few other linting rules. It is usually a good idea to lint locally as well using [flake8](https://flake8.readthedocs.org/en/latest/) using `flake8 airflow tests`. `git diff upstream/master -u -- "*.py" | flake8 --diff` will return any changed files in your branch that require linting.
+1. We also apply [Pylint](https://www.pylint.org) for linting (static code analysis). Run locally with `./scripts/ci/ci_pylint.sh`.
 1. Please read this excellent [article](http://chris.beams.io/posts/git-commit/) on commit messages and adhere to them. It makes the lives of those who come after you a lot easier.
 
 ### Testing on Travis CI

CONTRIBUTING.md

@@ -61,4 +61,4 @@
     document={"id": "someuniqueid", "param1": "value1", "param2": "value2"},
     azure_cosmos_conn_id='azure_cosmos_default')
 
-t1 >> t2
+t1 >> t2  # pylint: disable=pointless-statement

airflow/contrib/example_dags/example_azure_cosmosdb_sensor.py

@@ -212,12 +212,12 @@ def failure_callback(context):
     dag=dag,
 )
 
-[
+[  # pylint: disable=pointless-statement
     text_msg_remind_none,
     text_msg_remind_specific,
     text_msg_remind_include_invalid,
     text_msg_remind_all
-] >> link_msg >> markdown_msg >> [
+] >> link_msg >> markdown_msg >> [  # pylint: disable=pointless-statement
     single_action_card_msg,
     multi_action_card_msg
-] >> feed_card_msg >> msg_failure_callback
+] >> feed_card_msg >> msg_failure_callback  # pylint: disable=pointless-statement

airflow/contrib/example_dags/example_dingding_operator.py

@@ -46,6 +46,7 @@
 * CBT_POKE_INTERVAL - number of seconds between every attempt of Sensor check
 
 """
+
 import json
 
 from os import getenv
@@ -108,7 +109,7 @@
         cluster_storage_type=int(CBT_CLUSTER_STORAGE_TYPE),
         task_id='create_instance_task2',
     )
-    create_instance_task >> create_instance_task2
+    create_instance_task >> create_instance_task2  # pylint: disable=pointless-statement
     # [END howto_operator_gcp_bigtable_instance_create]
 
     # [START howto_operator_gcp_bigtable_cluster_update]
@@ -125,7 +126,7 @@
         nodes=int(CBT_CLUSTER_NODES_UPDATED),
         task_id='update_cluster_task2',
     )
-    cluster_update_task >> cluster_update_task2
+    cluster_update_task >> cluster_update_task2  # pylint: disable=pointless-statement
     # [END howto_operator_gcp_bigtable_cluster_update]
 
     # [START howto_operator_gcp_bigtable_instance_delete]
@@ -152,7 +153,7 @@
         table_id=CBT_TABLE_ID,
         task_id='create_table_task2',
     )
-    create_table_task >> create_table_task2
+    create_table_task >> create_table_task2  # pylint: disable=pointless-statement
     # [END howto_operator_gcp_bigtable_table_create]
 
     # [START howto_operator_gcp_bigtable_table_wait_for_replication]
@@ -187,6 +188,7 @@
     )
     # [END howto_operator_gcp_bigtable_table_delete]
 
+    # pylint: disable=pointless-statement
     wait_for_table_replication_task >> delete_table_task
     wait_for_table_replication_task2 >> delete_table_task
     wait_for_table_replication_task >> delete_table_task2
@@ -203,3 +205,4 @@
     # Only delete instances after all tables are deleted
     [delete_table_task, delete_table_task2] >> \
         delete_instance_task >> delete_instance_task2
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_bigtable_operators.py

@@ -29,6 +29,7 @@
 * GCE_SHORT_MACHINE_TYPE_NAME - Machine type resource name to set, e.g. 'n1-standard-1'.
     See https://cloud.google.com/compute/docs/machine-types
 """
+
 import os
 
 import airflow
@@ -110,5 +111,7 @@
     )
     # [END howto_operator_gce_set_machine_type_no_project_id]
 
+    # pylint: disable=pointless-statement
     gce_instance_start >> gce_instance_start2 >> gce_instance_stop >> \
         gce_instance_stop2 >> gce_set_machine_type >> gce_set_machine_type2
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_compute.py

@@ -139,6 +139,9 @@
             task_id='gcp_compute_igm_group_manager_update_template_2'
         )
     # [END howto_operator_gce_igm_update_template_no_project_id]
+
+    # pylint: disable=pointless-statement
     gce_instance_template_copy >> gce_instance_template_copy2 >> \
         gce_instance_group_manager_update_template >> \
         gce_instance_group_manager_update_template2
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_compute_igm.py

@@ -130,4 +130,4 @@
         name=FUNCTION_NAME
     )
     # [END howto_operator_gcf_delete]
-    deploy_task >> deploy2_task >> delete_task
+    deploy_task >> deploy2_task >> delete_task  # pylint: disable=pointless-statement

airflow/contrib/example_dags/example_gcp_function.py

@@ -21,7 +21,6 @@
 Example Airflow DAG for Google Cloud Natural Language service
 """
 
-
 from google.cloud.language_v1.proto.language_service_pb2 import Document
 
 import airflow
@@ -108,7 +107,9 @@
     )
     # [END howto_operator_gcp_natural_language_analyze_classify_text_result]
 
+    # pylint: disable=pointless-statement
     analyze_entities >> analyze_entities_result
     analyze_entity_sentiment >> analyze_entity_sentiment_result
     analyze_sentiment >> analyze_sentiment_result
     analyze_classify_text >> analyze_classify_text_result
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_natural_language.py

@@ -183,6 +183,7 @@
     )
     # [END howto_operator_spanner_delete]
 
+    # pylint: disable=pointless-statement
     spanner_instance_create_task \
         >> spanner_instance_update_task \
         >> spanner_database_deploy_task \
@@ -196,3 +197,4 @@
         >> spanner_database_delete_task2 \
         >> spanner_instance_delete_task \
         >> spanner_instance_delete_task2
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_spanner.py

@@ -84,7 +84,7 @@
     )
     # [END howto_operator_speech_to_text_recognize]
 
-    text_to_speech_synthesize_task >> speech_to_text_recognize_task
+    text_to_speech_synthesize_task >> speech_to_text_recognize_task  # pylint: disable=pointless-statement
 
     # [START howto_operator_translate_speech]
     translate_speech_task = GcpTranslateSpeechOperator(
@@ -99,4 +99,4 @@
     )
     # [END howto_operator_translate_speech]
 
-    text_to_speech_synthesize_task >> translate_speech_task
+    text_to_speech_synthesize_task >> translate_speech_task  # pylint: disable=pointless-statement

airflow/contrib/example_dags/example_gcp_speech.py

@@ -186,7 +186,7 @@
 ) as dag:
 
     def next_dep(task, prev):
-        prev >> task
+        prev >> task  # pylint: disable=pointless-statement
         return task
 
     # ############################################## #

airflow/contrib/example_dags/example_gcp_sql.py

@@ -287,7 +287,7 @@ def get_absolute_path(path):
         )
         tasks.append(task)
         if prev_task:
-            prev_task >> task
+            prev_task >> task  # pylint: disable=pointless-statement
         prev_task = task
 
 # [END howto_operator_cloudsql_query_operators]

airflow/contrib/example_dags/example_gcp_sql_query.py

@@ -38,6 +38,7 @@
   Look at documentation of :class:`~airflow.operators.sensors.BaseSensorOperator` for more information
 
 """
+
 import os
 from datetime import datetime, timedelta
 from typing import Any, Dict
@@ -247,7 +248,9 @@
         project_id=GCP_PROJECT_ID,
     )
 
+    # pylint: disable=pointless-statement
     create_transfer_job_from_aws >> wait_for_operation_to_start >> pause_operation >> \
         list_operations >> get_operation >> resume_operation >> wait_for_operation_to_end >> \
         create_transfer_job_from_gcp >> wait_for_second_operation_to_start >> cancel_operation >> \
         delete_transfer_from_aws_job >> delete_transfer_from_gcp_job
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_transfer.py

@@ -22,6 +22,7 @@
 service in the Google Cloud Platform.
 
 """
+
 import airflow
 from airflow import models
 
@@ -48,5 +49,5 @@
         task_id='access',
         bash_command="echo '{{ task_instance.xcom_pull(\"translate\")[0] }}'"
     )
-    product_set_create >> translation_access
+    product_set_create >> translation_access  # pylint: disable=pointless-statement
     # [END howto_operator_translate_access]

airflow/contrib/example_dags/example_gcp_translate.py

@@ -31,6 +31,7 @@
 * GCP_VISION_ANNOTATE_IMAGE_URL - A link to the bucket that contains the file to be annotated.
 
 """
+
 import os
 
 # [START howto_operator_vision_retry_import]
@@ -229,6 +230,7 @@
     )
     # [END howto_operator_vision_remove_product_from_product_set]
 
+    # pylint: disable=pointless-statement
     # Product path
     product_create >> product_get >> product_update >> product_delete
 
@@ -244,6 +246,7 @@
     add_product_to_product_set >> remove_product_from_product_set
     remove_product_from_product_set >> product_delete
     remove_product_from_product_set >> product_set_delete
+    # pylint: enable=pointless-statement
 
 with models.DAG(
     'example_gcp_vision_explicit_id', default_args=default_args, schedule_interval=None
@@ -381,6 +384,7 @@
     )
     # [END howto_operator_vision_remove_product_from_product_set_2]
 
+    # pylint: disable=pointless-statement
     # Product path
     product_create_2 >> product_create_2_idempotence >> product_get_2 >> product_update_2 >> product_delete_2
 
@@ -397,6 +401,7 @@
     product_create_2 >> add_product_to_product_set_2
     remove_product_from_product_set_2 >> product_set_delete_2
     remove_product_from_product_set_2 >> product_delete_2
+    # pylint: enable=pointless-statement
 
 with models.DAG(
     'example_gcp_vision_annotate_image', default_args=default_args, schedule_interval=None
@@ -476,9 +481,11 @@
     )
     # [END howto_operator_vision_detect_safe_search_result]
 
+    # pylint: disable=pointless-statement
     annotate_image >> annotate_image_result
 
     detect_text >> detect_text_result
     document_detect_text >> document_detect_text_result
     detect_labels >> detect_labels_result
     detect_safe_search >> detect_safe_search_result
+    # pylint: enable=pointless-statement

airflow/contrib/example_dags/example_gcp_vision.py

@@ -30,6 +30,7 @@
 * GCS_ACL_BUCKET_ROLE - The access permission for the entity for the bucket.
 * GCS_ACL_OBJECT_ROLE - The access permission for the entity for the object.
 """
+
 import os
 
 import airflow
@@ -73,4 +74,5 @@
     )
     # [END howto_operator_gcs_object_create_acl_entry_task]
 
+    # pylint: disable=pointless-statement
     gcs_bucket_create_acl_entry_task >> gcs_object_create_acl_entry_task

airflow/contrib/example_dags/example_gcs_acl.py

@@ -64,4 +64,4 @@
         bash_command='bq rm -rf airflow_test',
         dag=dag)
 
-    create_test_dataset >> load_csv >> delete_test_dataset
+    create_test_dataset >> load_csv >> delete_test_dataset  # pylint: disable=pointless-statement

airflow/contrib/example_dags/example_gcs_to_bq_operator.py

@@ -82,5 +82,5 @@
     t6 = PubSubSubscriptionDeleteOperator(task_id='delete-subscription')
     t7 = PubSubTopicDeleteOperator(task_id='delete-topic')
 
-    t1 >> t2 >> t3
-    t2 >> t4 >> t5 >> t6 >> t7
+    t1 >> t2 >> t3  # pylint: disable=pointless-statement
+    t2 >> t4 >> t5 >> t6 >> t7  # pylint: disable=pointless-statement

airflow/contrib/example_dags/example_pubsub_flow.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Script to run Pylint on all code. Should be started from root directory:
+# ./[airflow dir]/scripts/ci/ci_pylint.sh
+
+set -ex
+
+find . -name "*.py" \
+-not -path "./.eggs/*" \
+-not -path "./airflow/www/node_modules/*" \
+-not -path "./airflow/_vendor/*" \
+-not -path "./build" | grep -vFf scripts/ci/pylint_todo.txt | xargs pylint --output-format=colorized