Reorder tests to minimalize opened containers

Author: jasiu86

tests/functional/config.py

@@ -15,7 +15,6 @@
 #
 
 import os
-import sys
 
 from utils.helpers import get_int, get_bool
 from utils.parametrization import generate_test_object_name
@@ -117,6 +116,3 @@
 """ TT_SKIP_TEST_IF_HDDL_MTLS """
 skip_hddl_tests = get_bool("TT_SKIP_TEST_IF_HDDL_MTLS", "True")
 skip_hddl_tests = skip_hddl_tests and target_device == "HDDL"
-
-""" USING_XDIST """
-using_xdist = ('-n' in sys.argv) or ('-c' in sys.argv)

tests/functional/conftest.py

@@ -15,7 +15,6 @@
 #
 import logging
 import os
-import pickle
 from collections import defaultdict
 from logging import FileHandler
 
@@ -25,17 +24,16 @@
 from _pytest.outcomes import OutcomeException
 
 from constants import MODEL_SERVICE, PREDICTION_SERVICE
-from utils.helpers import get_xdist_worker_nr, get_xdist_worker_count
-from utils.xdist_utils import OvmsCLoadScheduling
 from object_model.server import Server
+from utils.other import reorder_items_by_fixtures_used
 from utils.cleanup import clean_hanging_docker_resources, delete_test_directory, \
     get_containers_with_tests_suffix, get_docker_client
 from utils.logger import init_logger
 from tensorflow_serving.apis import prediction_service_pb2_grpc, \
     model_service_pb2_grpc  # noqa
 from utils.files_operation import get_path_friendly_test_name
 from utils.parametrization import get_tests_suffix
-from config import test_dir, test_dir_cleanup, artifacts_dir, using_xdist
+from config import test_dir, test_dir_cleanup, artifacts_dir
 
 logger = logging.getLogger(__name__)
 
@@ -83,11 +81,6 @@ def pytest_configure():
     # Perform initial configuration.
     init_logger()
 
-    # if not os.environ.get("PYTEST_XDIST_WORKER", None):
-    #     copy_cached_models_to_test_dir()
-    #     copy_cached_resnet_models()
-
-
     init_conf_logger = logging.getLogger("init_conf")
 
     container_names = get_containers_with_tests_suffix()
@@ -121,45 +114,7 @@ def pytest_unconfigure():
 @pytest.hookimpl(hookwrapper=True)
 def pytest_collection_modifyitems(session, config, items):
     yield
-    items = OvmsCLoadScheduling.reorder_items_by_fixtures_used(session)
-
-
-#
-# @pytest.hookimpl(hookwrapper=True)
-# def pytest_collection_finish(session):
-#     yield
-#     # Collect all fixtures that starts Docker instance
-#     # This map will keep fixture usages in
-#
-
-
-# if not using_xdist:
-#     @pytest.hookimpl(hookwrapper=True)
-#     def pytest_runtestloop(session):
-#         # Override default runtestloop in order to sort test execution by used fixtures
-#         # This operation will ensure that only required containers will run and container will be cleared after all usages.
-#
-#         # Collect all fixtures that starts Docker instance
-#         # This map will keep fixture usages in tests
-#         server_fixtures_to_item = defaultdict(lambda: [])
-#         for item in session.items:
-#             item._server_fixtures = list(filter(lambda x: "start_server_" in x, item.fixturenames))
-#             for fixture in item._server_fixtures:
-#                 server_fixtures_to_item[fixture].append(item)
-#
-#         # Sort test items using required fixtures as key (group test by fixture)
-#         sorted_items = sorted(session.items, key=lambda x: x._server_fixtures )
-#
-#         for i, item in enumerate(sorted_items):
-#             nextitem = sorted_items[i + 1] if i + 1 < len(sorted_items) else None
-#             item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
-#
-#             # Test finished: remove test item for all fixtures that was used
-#             for fixture in item._server_fixtures:
-#                 server_fixtures_to_item[fixture].remove(item)
-#                 if len(server_fixtures_to_item[fixture]) == 0:
-#                     # No other tests will use this docker instance so we can close it.
-#                     Server.stop_by_fixture_name(fixture)
+    items = reorder_items_by_fixtures_used(session)
 
 
 @pytest.hookimpl(hookwrapper=True)
@@ -197,14 +152,13 @@ def pytest_runtest_teardown(item):
     yield
     # Test finished: remove test item for all fixtures that was used
     for fixture in item._server_fixtures:
-        if item in item.session._server_fixtures_to_item[fixture]:
-            item.session._server_fixtures_to_item[fixture].remove(item)
-        if len(item.session._server_fixtures_to_item[fixture]) == 0:
+        if item in item.session._server_fixtures_to_tests[fixture]:
+            item.session._server_fixtures_to_tests[fixture].remove(item)
+        if len(item.session._server_fixtures_to_tests[fixture]) == 0:
             # No other tests will use this docker instance so we can close it.
             Server.stop_by_fixture_name(fixture)
 
 
-
 def exception_catcher(when: str, outcome):
     if isinstance(outcome.excinfo, tuple):
         if len(outcome.excinfo) > 1 and isinstance(outcome.excinfo[1], OutcomeException):
@@ -241,8 +195,3 @@ def pytest_runtest_logfinish(nodeid, location):
         _root_logger.removeHandler(_root_logger._test_log_handler)
     yield
 
-
-def pytest_xdist_make_scheduler(config, log):
-    scheduler = OvmsCLoadScheduling(config, log)
-    log.debug("Created xdist scheduler")
-    return scheduler

tests/functional/fixtures/server_detection_model_fixtures.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from pathlib import Path
 
 import pytest
 
@@ -33,7 +32,7 @@ def start_server_face_detection_model_auto_shape(request):
     container_name_infix = "test-auto-shape"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -49,7 +48,7 @@ def start_server_face_detection_model_named_shape(request):
     container_name_infix = "test-named-shape"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -64,5 +63,5 @@ def start_server_face_detection_model_nonamed_shape(request):
     container_name_infix = "test-nonamed-shape"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()

tests/functional/fixtures/server_for_update_fixtures.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from pathlib import Path
 
 import pytest
 import shutil
@@ -38,7 +37,7 @@ def start_server_update_flow_latest(request):
     container_name_infix = "test-update-latest"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -55,5 +54,5 @@ def start_server_update_flow_specific(request):
     container_name_infix = "test-update-specific"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()

tests/functional/fixtures/server_local_models_fixtures.py

@@ -16,8 +16,6 @@
 
 import shutil
 import os
-from pathlib import Path
-
 import pytest
 
 import config
@@ -38,8 +36,7 @@ def start_server_single_model(request):
 
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    env_variables, target_device=config.target_device,
-                    path_to_mount=Path(config.path_to_mount, __name__))
+                    env_variables, target_device=config.target_device)
     return server.start()
 
 @pytest.fixture(scope="session")
@@ -55,22 +52,25 @@ def start_server_single_model_onnx(request):
 
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    env_variables, target_device=config.target_device,
-                    path_to_mount=Path(config.path_to_mount, __name__))
+                    env_variables, target_device=config.target_device)
     return server.start()
 
 @pytest.fixture(scope="session")
 def start_server_with_mapping(request):
-    path_to_mount = Path(config.path_to_mount, __name__)
 
-    file_dst_path = path_to_mount + '/age_gender/1/mapping_config.json'
-    Path(path_to_mount).mkdir(parents=True, exist_ok=True)
+    def delete_mapping_file():
+        if os.path.exists(file_dst_path):
+            os.remove(file_dst_path)
+
+    request.addfinalizer(delete_mapping_file)
+
+    file_dst_path = config.path_to_mount + '/age_gender/1/mapping_config.json'
     shutil.copyfile('tests/functional/mapping_config.json', file_dst_path)
 
     start_server_command_args = {"model_name": AgeGender.name,
                                  "model_path": AgeGender.model_path}
     container_name_infix = "test-2-out"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=path_to_mount)
+                    target_device=config.target_device)
     return server.start()

tests/functional/fixtures/server_multi_model_fixtures.py

@@ -15,8 +15,6 @@
 #
 
 import os
-from pathlib import Path
-
 import pytest
 
 import config
@@ -51,6 +49,5 @@ def start_server_multi_model(request, start_minio_server, get_minio_server_s3):
                                  "rest_workers": 2}
     container_name_infix = "test-multi"
     server = Server(request, start_server_command_args,
-                    container_name_infix, config.start_container_command, envs,
-                    path_to_mount=Path(config.path_to_mount, __name__))
+                    container_name_infix, config.start_container_command, envs)
     return server.start()

tests/functional/fixtures/server_remote_models_fixtures.py

@@ -15,7 +15,6 @@
 #
 
 import os
-from pathlib import Path
 
 import boto3
 import pytest
@@ -42,7 +41,7 @@ def start_server_single_model_from_gc(request):
     envs = ['https_proxy=' + os.getenv('https_proxy', "")]
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command, envs,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -164,5 +163,5 @@ def start_server_single_model_from_minio(request, get_minio_server_s3):
     container_name_infix = "test-single-minio"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command, envs,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()

tests/functional/fixtures/server_with_batching_fixtures.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from pathlib import Path
 
 import pytest
 
@@ -29,7 +28,7 @@ def start_server_batch_model(request):
     container_name_infix = "test-batch"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -41,7 +40,7 @@ def start_server_batch_model_2out(request):
     container_name_infix = "test-batch-2out"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -54,7 +53,7 @@ def start_server_batch_model_auto(request):
     container_name_infix = "test-autobatch"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -67,7 +66,7 @@ def start_server_batch_model_auto_2out(request):
     container_name_infix = "test-autobatch-2out"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -80,7 +79,7 @@ def start_server_batch_model_bs4(request):
     container_name_infix = "test-batch4"
     server = Server(request, start_server_command_args,
                     container_name_infix, config.start_container_command,
-                    target_device=config.target_device, path_to_mount=Path(config.path_to_mount, __name__))
+                    target_device=config.target_device)
     return server.start()
 
 
@@ -92,6 +91,5 @@ def start_server_batch_model_auto_bs4_2out(request):
                                  "batch_size": 4}
     container_name_infix = "test-batch4-2out"
     server = Server(request, start_server_command_args,
-                    container_name_infix, config.start_container_command,
-                    path_to_mount=Path(config.path_to_mount, __name__))
+                    container_name_infix, config.start_container_command)
     return server.start()

tests/functional/fixtures/server_with_version_policy_fixtures.py

@@ -15,8 +15,6 @@
 #
 
 import os
-from pathlib import Path
-
 import pytest
 import shutil
 from distutils.dir_util import copy_tree
@@ -36,8 +34,7 @@ def start_server_model_ver_policy(request):
     container_name_infix = "test-batch4-2out"
 
     server = Server(request, start_server_command_args,
-                    container_name_infix, config.start_container_command,
-                    path_to_mount=Path(config.path_to_mount, __name__))
+                    container_name_infix, config.start_container_command)
     return server.start()
 
 

tests/functional/object_model/docker.py

@@ -15,15 +15,13 @@
 #
 import os
 import time
-from pathlib import Path
 from typing import List
 
 from datetime import datetime
 
 import docker
 from utils.files_operation import get_path_friendly_test_name
 
-from config import target_device
 from retry.api import retry_call
 
 import config
@@ -126,12 +124,6 @@ def _start(self):
         self.ensure_logs_contains()
         logger.info(f"Container started grpc_port:{self.grpc_port}\trest_port:{self.rest_port}")
         logger.debug(f"Container starting command args: {self.start_container_command}")
-
-        tt = list(map(lambda x: x.name, self.client.containers.list()))
-        tt = list(map(lambda x: "_".join(x.split("_")[2:-3]), tt))
-
-        foo = 0
-
         return self.container, {"grpc_port": self.grpc_port, "rest_port": self.rest_port}
 
     def stop(self):
@@ -202,7 +194,7 @@ def save_container_logs_to_file(self, logs, dir_path: str = config.artifacts_dir
         if location:
             file_name = f"ovms_{get_path_friendly_test_name(location)}_{time_stamp}.log"
         else:
-            file_name = f"ovms_{self.server.started_by_fixture}_{time_stamp}.log"
+            file_name = f"ovms_{self.server.started_by_fixture.lstrip('start_')}_{time_stamp}.log"
         os.makedirs(dir_path, exist_ok=True)
         file_path = os.path.join(dir_path, file_name)
         with open(file_path, "w+") as text_file: