Formatting code using ruff

Author: rpmcginty

src/aibs_informatics_aws_lambda/common/api/handler.py

@@ -5,19 +5,15 @@
 import logging
 from dataclasses import dataclass, field
 from datetime import datetime
-from types import ModuleType
-from typing import Any, Callable, Dict, Generic, List, Optional, Type, TypeVar, Union, cast
+from typing import Any, Callable, Dict, Generic, Optional, TypeVar, Union, cast
 
 from aibs_informatics_core.models.api.http_parameters import HTTPParameters
 from aibs_informatics_core.models.api.route import ApiRoute
 from aibs_informatics_core.models.base import ModelProtocol
 from aibs_informatics_core.utils.json import JSON
-from aibs_informatics_core.utils.modules import get_all_subclasses, load_all_modules_from_pkg
-from aws_lambda_powertools.event_handler import APIGatewayRestResolver, content_types
 from aws_lambda_powertools.event_handler.api_gateway import BaseRouter
 from aws_lambda_powertools.logging import Logger
 from aws_lambda_powertools.metrics import EphemeralMetrics, Metrics
-from aws_lambda_powertools.tracing import Tracer
 from aws_lambda_powertools.utilities.data_classes.api_gateway_proxy_event import (
     APIGatewayEventRequestContext,
     APIGatewayProxyEvent,
@@ -28,11 +24,8 @@
 )
 from aws_lambda_powertools.utilities.typing import LambdaContext
 
-from aibs_informatics_aws_lambda.common.base import HandlerMixins
 from aibs_informatics_aws_lambda.common.handler import LambdaHandler
-from aibs_informatics_aws_lambda.common.logging import LoggingMixins
 from aibs_informatics_aws_lambda.common.metrics import (
-    MetricsMixins,
     add_duration_metric,
     add_failure_metric,
     add_success_metric,
@@ -101,7 +94,6 @@ def add_to_router(
         logger = logger or cls.get_logger(service=cls.service_name())
         metrics = metrics or cls.get_metrics()
 
-        # TODO: remove args once https://github.com/python/mypy/pull/15133 is released (should be mypy 1.2.1)
         @metrics.log_metrics
         @router.route(rule=cls.route_rule(), method=cls.route_method())
         def gateway_handler(logger=logger, metrics=metrics, **route_parameters) -> Any:
@@ -128,7 +120,7 @@ def gateway_handler(logger=logger, metrics=metrics, **route_parameters) -> Any:
                     *args, _current_event=router.current_event, **kwargs
                 )
 
-                logger.info(f"Route handler method constructed. Invoking")
+                logger.info("Route handler method constructed. Invoking")
                 response = lambda_handler(event, router.lambda_context)
                 add_success_metric(metrics=metrics)
                 add_duration_metric(start=start, metrics=metrics)
@@ -144,7 +136,7 @@ def gateway_handler(logger=logger, metrics=metrics, **route_parameters) -> Any:
     def _parse_event(
         cls, event: BaseProxyEvent, route_parameters: Dict[str, Any], logger: logging.Logger
     ) -> API_REQUEST:
-        logger.info(f"parsing event.")
+        logger.info("parsing event.")
         stringified_route_params = route_parameters
         stringified_query_params = event.query_string_parameters
         stringified_request_body = event.json_body if event.body else None
@@ -162,7 +154,7 @@ def _parse_event(
         )
         logger.debug(f"Constructed following HTTP Parameters: {http_parameters}")
 
-        logger.debug(f"Converting HTTP Parameters to request object")
+        logger.debug("Converting HTTP Parameters to request object")
         request = cls.get_request_from_http_parameters(http_parameters)
         return request
 

src/aibs_informatics_aws_lambda/common/api/resolver.py

@@ -18,7 +18,6 @@
 from aws_lambda_powertools.logging import Logger
 from aws_lambda_powertools.logging.correlation_paths import API_GATEWAY_REST
 from aws_lambda_powertools.metrics import EphemeralMetrics, Metrics
-from aws_lambda_powertools.tracing import Tracer
 from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEvent
 from aws_lambda_powertools.utilities.typing import LambdaContext
 

src/aibs_informatics_aws_lambda/common/handler.py

@@ -244,4 +244,9 @@ def handler(event, context: LambdaContext):
         return handler  # type: ignore
 
     def __repr__(self) -> str:
-        return f"{self.__class__.__name__}(request: {self.get_request_cls()}, response: {self.get_response_cls()})"
+        return (
+            f"{self.__class__.__name__}("
+            f"request: {self.get_request_cls()}, "
+            f"response: {self.get_response_cls()}"
+            ")"
+        )

src/aibs_informatics_aws_lambda/common/models.py

@@ -1,6 +1,6 @@
 import inspect
 from dataclasses import dataclass, field
-from typing import Optional, cast
+from typing import cast
 
 import marshmallow as mm
 from aibs_informatics_aws_utils.constants.lambda_ import (

src/aibs_informatics_aws_lambda/handlers/batch/model.py

@@ -33,16 +33,16 @@ class CreateDefinitionAndPrepareArgsRequest(SchemaModel):
     command: List[str] = custom_field(default_factory=list)
     environment: Dict[str, str] = custom_field(default_factory=dict)
     job_definition_tags: Dict[str, str] = custom_field(default_factory=dict)
-    resource_requirements: Union[
-        List[ResourceRequirementTypeDef], ResourceRequirements
-    ] = custom_field(
-        default_factory=list,
-        mm_field=UnionField(
-            [
-                (list, ListField(DictField)),
-                (ResourceRequirements, ResourceRequirements.as_mm_field()),
-            ]
-        ),
+    resource_requirements: Union[List[ResourceRequirementTypeDef], ResourceRequirements] = (
+        custom_field(
+            default_factory=list,
+            mm_field=UnionField(
+                [
+                    (list, ListField(DictField)),
+                    (ResourceRequirements, ResourceRequirements.as_mm_field()),
+                ]
+            ),
+        )
     )
     mount_points: List[MountPointTypeDef] = custom_field(default_factory=list)
     volumes: List[VolumeTypeDef] = custom_field(default_factory=list)

src/aibs_informatics_aws_lambda/handlers/data_sync/file_system.py

@@ -2,7 +2,6 @@
 from datetime import timedelta
 from pathlib import Path
 from typing import List, TypeVar
-from xml.etree.ElementInclude import include
 
 from aibs_informatics_aws_utils.data_sync.file_system import BaseFileSystem, Node, get_file_system
 from aibs_informatics_aws_utils.efs import detect_mount_points, get_local_path

src/aibs_informatics_aws_lambda/handlers/data_sync/model.py

@@ -24,15 +24,18 @@
 from aibs_informatics_core.utils.time import get_current_time
 
 DataPath = Union[S3Path, EFSPath, Path, str]
-DataPathField = lambda *args, **kwargs: UnionField(
-    [
-        (S3Path, S3Path.as_mm_field()),
-        ((EFSPath, str), EFSPath.as_mm_field()),
-        ((Path, str), PathField()),
-    ],
-    *args,
-    **kwargs,
-)
+
+
+def DataPathField(*args, **kwargs):
+    return UnionField(
+        [
+            (S3Path, S3Path.as_mm_field()),
+            ((EFSPath, str), EFSPath.as_mm_field()),
+            ((Path, str), PathField()),
+        ],
+        *args,
+        **kwargs,
+    )
 
 
 @dataclass

src/aibs_informatics_aws_lambda/handlers/data_sync/operations.py

@@ -70,10 +70,10 @@ def handle(self, request: GetJSONFromFileRequest) -> GetJSONFromFileResponse:
 
             self.logger.info(f"Fetching content from {path}")
             if isinstance(path, S3URI):
-                self.logger.info(f"Downloading from S3")
+                self.logger.info("Downloading from S3")
                 content = download_to_json(s3_path=path)
             else:
-                self.logger.info(f"Loading from path")
+                self.logger.info("Loading from path")
                 content = load_json(path)
             return GetJSONFromFileResponse(content=content)
         except Exception as e:
@@ -103,10 +103,10 @@ def handle(self, request: PutJSONToFileRequest) -> Optional[PutJSONToFileRespons
         self.logger.info(f"Writing content to {path}")
         self.logger.info(f"Content to write: {content}")
         if isinstance(path, S3URI):
-            self.logger.info(f"Uploading to S3")
+            self.logger.info("Uploading to S3")
             upload_json(content, s3_path=path, extra_args=SCRATCH_EXTRA_ARGS)
         else:
-            self.logger.info(f"Writing to file")
+            self.logger.info("Writing to file")
             path.parent.mkdir(parents=True, exist_ok=True)
             path.write_text(json.dumps(content, indent=4, sort_keys=True))
         return PutJSONToFileResponse(path=path)
@@ -136,7 +136,7 @@ def handle(self, request: BatchDataSyncRequest) -> BatchDataSyncResponse:
         for i, _ in enumerate(batch_requests):
             sync_operations = DataSyncOperations(_)
             self.logger.info(
-                f"[{i+1}/{len(batch_requests)}] "
+                f"[{i + 1}/{len(batch_requests)}] "
                 f"Syncing content from {_.source_path} to {_.destination_path}"
             )
             try:
@@ -251,9 +251,9 @@ def build_destination_path(
         if node.has_children():
             relative_path += "/"
         if isinstance(request.destination_path, S3URI):
-            return S3URI.build(
-                bucket_name=request.destination_path.bucket,
-                key=request.destination_path.key + relative_path,
+            return S3URI(
+                f"s3://{request.destination_path.bucket}/"
+                f"{request.destination_path.key + relative_path}"
             )
         else:
             return Path(f"{request.destination_path}/{relative_path}")
@@ -283,10 +283,11 @@ def build_node_batches(
         ## We will use a revised version of the bin packing problem:
         # https://en.wikipedia.org/wiki/Bin_packing_problem
 
-        # Step 1: and then sort the nodes by size (descending order)
+        # Step 1:   and then sort the nodes by size (descending order)
         unbatched_nodes = sorted(nodes, key=lambda node: node.size_bytes, reverse=True)
 
-        # Step 2: Group nodes in order to maximize the data synced per request (bin packing problem)
+        # Step 2:   Group nodes in order to maximize the data synced per request
+        #           (bin packing problem)
         node_batches: List[List[Node]] = []
 
         # (Optimize) Convert all nodes that are larger than the threshold into single requests.

src/aibs_informatics_aws_lambda/handlers/demand/context_manager.py

@@ -65,10 +65,12 @@ class BatchEFSConfiguration:
     volume: VolumeTypeDef = field(init=False)
 
     def __post_init__(self) -> None:
+        file_system = self.mount_point_config.file_system
+        name_or_id = file_system.get("Name", file_system["FileSystemId"])
         volume_name = "-".join(
             [
-                f'{self.mount_point_config.file_system.get("Name", self.mount_point_config.file_system["FileSystemId"])}',
-                f'{str(self.mount_path).strip("/").replace("/","-")}-vol',
+                f"{name_or_id}",
+                f"{str(self.mount_path).strip('/').replace('/', '-')}-vol",
             ]
         )
 
@@ -293,7 +295,7 @@ def post_execution_data_sync_requests(self) -> List[PrepareBatchDataSyncRequest]
                 )
             )
         logger.info(
-            f"Generated {len(requests)} data sync requests for post-execution data sync: {requests}"
+            f"Generated {len(requests)} data sync requests for post-execution data sync: {requests}"  # noqa: E501
         )
         return requests
 
@@ -488,7 +490,7 @@ def get_batch_efs_configuration(
     )
 
 
-def generate_batch_job_builder(
+def generate_batch_job_builder(  # noqa: C901
     demand_execution: DemandExecution,
     env_base: EnvBase,
     working_path: EFSPath,
@@ -498,7 +500,7 @@ def generate_batch_job_builder(
     tmp_mount_point: Optional[MountPointConfiguration] = None,
     env_file_write_mode: EnvFileWriteMode = EnvFileWriteMode.ALWAYS,
 ) -> BatchJobBuilder:
-    logger.info(f"Constructing BatchJobBuilder instance")
+    logger.info("Constructing BatchJobBuilder instance")
 
     demand_execution = demand_execution.copy()
     efs_mount_points = [scratch_mount_point, shared_mount_point]
@@ -585,8 +587,9 @@ def generate_batch_job_builder(
 
             if env_size > 8192 * 0.9:
                 logger.info(
-                    f"Environment variables are too large to pass directly to container (> 90% of 8192). "
-                    f"Writing environment variables to file {efs_environment_file_uri}."
+                    f"Environment variables are too large to pass directly to container "
+                    "(> 90% of 8192). Writing environment variables to file "
+                    f"{efs_environment_file_uri}."
                 )
                 confirm_write = True
             else:
@@ -632,7 +635,7 @@ def generate_batch_job_builder(
     ]
     if tmp_mount_point:
         vol_configurations.append(BatchEFSConfiguration(tmp_mount_point, read_only=False))
-    logger.info(f"Constructing BatchJobBuilder instance...")
+    logger.info("Constructing BatchJobBuilder instance...")
     return BatchJobBuilder(
         image=demand_execution.execution_image,
         job_definition_name=env_base.get_job_name(

src/aibs_informatics_aws_lambda/handlers/demand/scaffolding.py

@@ -124,7 +124,7 @@ def setup_file_system(self, context_manager: DemandExecutionContextManager):
         Args:
             context_manager (DemandExecutionContextManager): context manager
         """
-        working_path = context_manager.container_working_path
+        working_path = context_manager.container_working_path  # noqa: F841
         # working_path.mkdir(parents=True, exist_ok=True)