Implements most black upgrade format fixes

Author: V

airflow/dags/airtable_loader_v2/generate_gtfs_download_configs.py

@@ -49,18 +49,22 @@ def gtfs_datasets_to_extract_configs(
                         name=record.name,
                         url=record.pipeline_url,
                         feed_type=record.data,
-                        auth_query_params={
-                            record.authorization_url_parameter_name: record.url_secret_key_name
-                        }
-                        if record.authorization_url_parameter_name
-                        and record.url_secret_key_name
-                        else {},
-                        auth_headers={
-                            record.authorization_header_parameter_name: record.header_secret_key_name
-                        }
-                        if record.authorization_header_parameter_name
-                        and record.header_secret_key_name
-                        else {},
+                        auth_query_params=(
+                            {
+                                record.authorization_url_parameter_name: record.url_secret_key_name
+                            }
+                            if record.authorization_url_parameter_name
+                            and record.url_secret_key_name
+                            else {}
+                        ),
+                        auth_headers=(
+                            {
+                                record.authorization_header_parameter_name: record.header_secret_key_name
+                            }
+                            if record.authorization_header_parameter_name
+                            and record.header_secret_key_name
+                            else {}
+                        ),
                     ),
                 )
             except ValidationError as e:

airflow/dags/dags.py

@@ -55,9 +55,11 @@ def log_failure_to_slack(context):
         wait_for_defaults={"retries": 24, "check_existence": True, "timeout": 10 * 60},
         latest_only=False,
         user_defined_macros={
-            "image_tag": lambda: "development"
-            if os.environ["AIRFLOW_ENV"] == "development"
-            else "latest",
+            "image_tag": lambda: (
+                "development"
+                if os.environ["AIRFLOW_ENV"] == "development"
+                else "latest"
+            ),
             "env_var": os.environ.get,
         },
         default_args={

airflow/plugins/operators/external_table.py

@@ -7,6 +7,7 @@
 However, it's cumbersome to convert the http api style schema fields to SQL, so
 we provide a fallback for these old-style tasks.
 """
+
 from google.api_core.exceptions import NotFound
 from google.cloud import bigquery
 from utils import CALITP_BQ_LOCATION, CALITP_PROJECT_NAME

airflow/plugins/scripts/gtfs_rt_parser.py

@@ -1,6 +1,7 @@
 """
 Parses binary RT feeds and writes them back to GCS as gzipped newline-delimited JSON
 """
+
 import base64
 import copy
 import datetime
@@ -414,9 +415,9 @@ def where_base64url(self, base64_url: Optional[str]):
     def get_aggregates(
         self,
     ) -> List[RTHourlyAggregation]:
-        aggregates: Dict[
-            Tuple[pendulum.DateTime, str], List[GTFSRTFeedExtract]
-        ] = defaultdict(list)
+        aggregates: Dict[Tuple[pendulum.DateTime, str], List[GTFSRTFeedExtract]] = (
+            defaultdict(list)
+        )
 
         for file in self.files:
             if self.base64_url is None or file.base64_url == self.base64_url:
@@ -892,9 +893,9 @@ def parse_and_validate(
 
 def make_dict_bq_safe(d: Dict[str, Any]) -> Dict[str, Any]:
     return {
-        make_name_bq_safe(key): make_dict_bq_safe(value)
-        if isinstance(value, dict)
-        else value
+        make_name_bq_safe(key): (
+            make_dict_bq_safe(value) if isinstance(value, dict) else value
+        )
         for key, value in d.items()
     }
 

packages/calitp-data-analysis/calitp_data_analysis/geography_utils.py

@@ -2,6 +2,7 @@
 Utility functions for geospatial data.
 Some functions for dealing with census tract or other geographic unit dfs.
 """
+
 from typing import Literal, Union, cast
 
 import dask.dataframe as dd

packages/calitp-data-analysis/calitp_data_analysis/styleguide.py

@@ -24,6 +24,7 @@
 https://github.com/CityOfLosAngeles/los-angeles-citywide-data-style
 
 """
+
 import altair as alt  # type: ignore
 from calitp_data_analysis import calitp_color_palette as cp
 

packages/calitp-data-analysis/calitp_data_analysis/utils.py

@@ -1,6 +1,7 @@
 """
 General utility functions.
 """
+
 import base64
 import os
 import shutil

packages/calitp-data-infra/calitp_data_infra/storage.py

@@ -643,9 +643,9 @@ def build_request(self, auth_dict: Mapping[str, str]) -> Request:
         headers = {k: auth_dict[v] for k, v in self.auth_headers.items()}
 
         # some web servers require user agents or they will throw a 4XX error
-        headers[
-            "User-Agent"
-        ] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0"
+        headers["User-Agent"] = (
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0"
+        )
 
         # inspired by: https://stackoverflow.com/questions/18869074/create-url-without-request-execution
         return Request(

warehouse/scripts/dbt_artifacts/__init__.py

@@ -1,6 +1,7 @@
 """
 Built off the starting point of https://guitton.co/posts/dbt-artifacts
 """
+
 import abc
 import os
 from enum import Enum
@@ -64,9 +65,9 @@ def num_bytes(self) -> Optional[int]:
 CatalogTable.num_bytes = property(num_bytes)  # type: ignore[attr-defined]
 
 DependsOn.resolved_nodes = property(  # type: ignore[attr-defined]
-    lambda self: [NodeModelMixin._instances[node] for node in self.nodes]
-    if self.nodes
-    else []
+    lambda self: (
+        [NodeModelMixin._instances[node] for node in self.nodes] if self.nodes else []
+    )
 )
 ColumnInfo.publish = property(lambda self: self.meta.get("publish.include", False))  # type: ignore[attr-defined]
 

warehouse/scripts/dbt_artifacts/manifest.py

@@ -24,9 +24,9 @@ class ManifestMetadata(BaseModel):
     class Config:
         extra = Extra.allow
 
-    dbt_schema_version: Optional[
-        str
-    ] = "https://schemas.getdbt.com/dbt/manifest/v9.json"
+    dbt_schema_version: Optional[str] = (
+        "https://schemas.getdbt.com/dbt/manifest/v9.json"
+    )
     dbt_version: Optional[str] = "1.6.0a1"
     generated_at: Optional[datetime] = "2023-04-21T11:09:06.496436Z"
     invocation_id: Optional[str] = "c4b245be-8edb-4ad7-ba54-9337ce594f5d"