diff --git a/ydb/tests/compatibility/test_compatibility.py b/ydb/tests/compatibility/test_compatibility.py index b47c2b1e0f8c..4595831fb72e 100644 --- a/ydb/tests/compatibility/test_compatibility.py +++ b/ydb/tests/compatibility/test_compatibility.py @@ -2,81 +2,26 @@ import pytest import yatest import os -import time -from ydb.tests.library.harness.kikimr_runner import KiKiMR -from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator -from ydb.tests.library.harness.param_constants import kikimr_driver_path -from ydb.tests.library.common.types import Erasure +from ydb.tests.library.compatibility.fixtures import RestartToAnotherVersionFixture from ydb.tests.oss.ydb_sdk_import import ydb from decimal import Decimal -last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/ydbd-last-stable") -current_binary_path = kikimr_driver_path() - -all_binary_combinations = [ - [[last_stable_binary_path], [current_binary_path]], - [[last_stable_binary_path], [last_stable_binary_path, current_binary_path]], - [[current_binary_path], [last_stable_binary_path]], - [[current_binary_path], [current_binary_path]], -] -all_binary_combinations_ids = [ - "last_stable_to_current", - "last_stable_to_current_mixed", - "current_to_last_stable", - "current_to_current", -] - - -class TestCompatibility(object): - @pytest.fixture(autouse=True, params=all_binary_combinations, ids=all_binary_combinations_ids) - def setup(self, request): - self.all_binary_paths = request.param - self.config = KikimrConfigGenerator( - erasure=Erasure.MIRROR_3_DC, - binary_paths=self.all_binary_paths[0], - use_in_memory_pdisks=False, - +class TestCompatibility(RestartToAnotherVersionFixture): + @pytest.fixture(autouse=True, scope="function") + def setup(self): + output_path = yatest.common.test_output_path() + self.output_f = open(os.path.join(output_path, "out.log"), "w") + yield from self.setup_cluster( extra_feature_flags={ "suppress_compatibility_check": True, # "enable_table_datetime64": True # uncomment for 64 datetime in tpc-h/tpc-ds }, column_shard_config={ 'disabled_on_scheme_shard': False, - }, - ) - - self.cluster = KiKiMR(self.config) - self.cluster.start() - self.endpoint = "grpc://%s:%s" % ('localhost', self.cluster.nodes[1].port) - output_path = yatest.common.test_output_path() - self.output_f = open(os.path.join(output_path, "out.log"), "w") - - self.driver = ydb.Driver( - ydb.DriverConfig( - database='/Root', - endpoint=self.endpoint - ) - ) - self.driver.wait() - yield - self.cluster.stop() - - def change_cluster_version(self, new_binary_paths): - self.config.set_binary_paths(new_binary_paths) - self.cluster.update_configurator_and_restart(self.config) - self.driver = ydb.Driver( - ydb.DriverConfig( - database='/Root', - endpoint=self.endpoint - ) + } ) - self.driver.wait() - # TODO: remove sleep - # without sleep there are errors like - # ydb.issues.Unavailable: message: "Failed to resolve tablet: 72075186224037909 after several retries." severity: 1 (server_code: 400050) - time.sleep(60) def execute_scan_query(self, query_body): query = ydb.ScanQuery(query_body, {}) @@ -139,7 +84,7 @@ def create_table(self, store_type): create_table(self, store_type) upsert_and_check_sum(self) - self.change_cluster_version(self.all_binary_paths[1]) + self.change_cluster_version() assert self.execute_scan_query('select count(*) as row_count from `sample_table`')[0]['row_count'] == 200, 'Expected 200 rows after update version' upsert_and_check_sum(self, iteration_count=2, start_index=100) assert self.execute_scan_query('select count(*) as row_count from `sample_table`')[0]['row_count'] == 500, 'Expected 500 rows: update 100-200 rows and added 300 rows' @@ -215,6 +160,6 @@ def test_tpch1(self, store_type): yatest.common.execute(init_command, wait=True, stdout=self.output_f) yatest.common.execute(import_command, wait=True, stdout=self.output_f) yatest.common.execute(run_command, wait=True, stdout=self.output_f) - self.change_cluster_version(self.all_binary_paths[1]) + self.change_cluster_version() yatest.common.execute(run_command, wait=True, stdout=self.output_f) yatest.common.execute(clean_command, wait=True, stdout=self.output_f) diff --git a/ydb/tests/compatibility/test_export_s3.py b/ydb/tests/compatibility/test_export_s3.py index 2ba1c4950e57..d119e7a2ac42 100644 --- a/ydb/tests/compatibility/test_export_s3.py +++ b/ydb/tests/compatibility/test_export_s3.py @@ -13,7 +13,7 @@ from ydb.tests.oss.ydb_sdk_import import ydb -last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/ydbd-last-stable") +last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/binaries/ydbd-last-stable") current_binary_path = kikimr_driver_path() all_binary_combinations = [ diff --git a/ydb/tests/compatibility/test_followers.py b/ydb/tests/compatibility/test_followers.py index 7b931eee2913..ab8615811c5e 100644 --- a/ydb/tests/compatibility/test_followers.py +++ b/ydb/tests/compatibility/test_followers.py @@ -16,7 +16,7 @@ class TestFollowersCompatibility(object): @classmethod def setup_class(cls): - last_stable_path = yatest.common.binary_path("ydb/tests/library/compatibility/ydbd-last-stable") + last_stable_path = yatest.common.binary_path("ydb/tests/library/compatibility/binaries/ydbd-last-stable") binary_paths = [kikimr_driver_path(), last_stable_path] cls.datacenters = [1, 2, 3] cls.dc_map = {i : cls.datacenters[(i - 1) % 3] for i in range(1, 10)} diff --git a/ydb/tests/compatibility/test_stress.py b/ydb/tests/compatibility/test_stress.py index 53615be0740b..9fccc0a3d463 100644 --- a/ydb/tests/compatibility/test_stress.py +++ b/ydb/tests/compatibility/test_stress.py @@ -11,7 +11,7 @@ from ydb.tests.library.common.types import Erasure from ydb.tests.stress.simple_queue.workload import Workload -last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/ydbd-last-stable") +last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/binaries/ydbd-last-stable") current_binary_path = kikimr_driver_path() all_binary_combinations = [ diff --git a/ydb/tests/compatibility/ya.make b/ydb/tests/compatibility/ya.make index d357e11b423d..5e2dce1857ae 100644 --- a/ydb/tests/compatibility/ya.make +++ b/ydb/tests/compatibility/ya.make @@ -23,13 +23,14 @@ INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/s3_recipe/recipe.inc) DEPENDS( ydb/apps/ydb ydb/apps/ydbd - ydb/tests/library/compatibility + ydb/tests/library/compatibility/binaries ) PEERDIR( contrib/python/boto3 ydb/tests/library ydb/tests/stress/simple_queue/workload + ydb/tests/library/compatibility ) END() diff --git a/ydb/tests/library/compatibility/downloader/__main__.py b/ydb/tests/library/compatibility/binaries/downloader/__main__.py similarity index 100% rename from ydb/tests/library/compatibility/downloader/__main__.py rename to ydb/tests/library/compatibility/binaries/downloader/__main__.py diff --git a/ydb/tests/library/compatibility/downloader/ya.make b/ydb/tests/library/compatibility/binaries/downloader/ya.make similarity index 100% rename from ydb/tests/library/compatibility/downloader/ya.make rename to ydb/tests/library/compatibility/binaries/downloader/ya.make diff --git a/ydb/tests/library/compatibility/binaries/ya.make b/ydb/tests/library/compatibility/binaries/ya.make new file mode 100644 index 000000000000..f4c89d695efd --- /dev/null +++ b/ydb/tests/library/compatibility/binaries/ya.make @@ -0,0 +1,15 @@ +RECURSE(downloader) + +UNION() + +RUN_PROGRAM( + ydb/tests/library/compatibility/binaries/downloader stable-25-1/release/ydbd ydbd-last-stable + OUT_NOAUTO ydbd-last-stable +) + +RUN_PROGRAM( + ydb/tests/library/compatibility/binaries/downloader stable-24-4/release/ydbd ydbd-prelast-stable + OUT_NOAUTO ydbd-prelast-stable +) + +END() diff --git a/ydb/tests/library/compatibility/fixtures.py b/ydb/tests/library/compatibility/fixtures.py new file mode 100644 index 000000000000..d0b1feff25cf --- /dev/null +++ b/ydb/tests/library/compatibility/fixtures.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +import pytest +import yatest +import time +from ydb.tests.library.harness.kikimr_runner import KiKiMR +from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator +from ydb.tests.library.harness.param_constants import kikimr_driver_path +from ydb.tests.library.common.types import Erasure +from ydb.tests.oss.ydb_sdk_import import ydb + + +last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/binaries/ydbd-last-stable") +current_binary_path = kikimr_driver_path() + +all_binary_combinations = [ + [[last_stable_binary_path], [current_binary_path]], + [[last_stable_binary_path], [last_stable_binary_path, current_binary_path]], + [[current_binary_path], [last_stable_binary_path]], + [[current_binary_path], [current_binary_path]], +] +all_binary_combinations_ids = [ + "last_stable_to_current", + "last_stable_to_current_mixed", + "current_to_last_stable", + "current_to_current", +] + + +class RestartToAnotherVersionFixture: + @pytest.fixture(autouse=True, params=all_binary_combinations, ids=all_binary_combinations_ids) + def base_setup(self, request): + self.current_binary_paths_index = 0 + self.all_binary_paths = request.param + + def setup_cluster(self, **kwargs): + self.config = KikimrConfigGenerator( + erasure=Erasure.MIRROR_3_DC, + binary_paths=self.all_binary_paths[self.current_binary_paths_index], + use_in_memory_pdisks=False, + **kwargs, + ) + + self.cluster = KiKiMR(self.config) + self.cluster.start() + self.endpoint = "grpc://%s:%s" % ('localhost', self.cluster.nodes[1].port) + + self.driver = ydb.Driver( + ydb.DriverConfig( + database='/Root', + endpoint=self.endpoint + ) + ) + self.driver.wait() + yield + self.cluster.stop() + + def change_cluster_version(self): + self.current_binary_paths_index = (self.current_binary_paths_index + 1) % len(self.all_binary_paths) + new_binary_paths = self.all_binary_paths[self.current_binary_paths_index] + self.config.set_binary_paths(new_binary_paths) + self.cluster.update_configurator_and_restart(self.config) + self.driver = ydb.Driver( + ydb.DriverConfig( + database='/Root', + endpoint=self.endpoint + ) + ) + self.driver.wait() + # TODO: remove sleep + # without sleep there are errors like + # ydb.issues.Unavailable: message: "Failed to resolve tablet: 72075186224037909 after several retries." severity: 1 (server_code: 400050) + time.sleep(60) diff --git a/ydb/tests/library/compatibility/ya.make b/ydb/tests/library/compatibility/ya.make index 42219b27eca7..37f21d79a083 100644 --- a/ydb/tests/library/compatibility/ya.make +++ b/ydb/tests/library/compatibility/ya.make @@ -1,15 +1,10 @@ -UNION() +RECURSE(binaries) + +PY23_LIBRARY() -RUN_PROGRAM( - ydb/tests/library/compatibility/downloader stable-25-1/release/ydbd ydbd-last-stable - OUT_NOAUTO ydbd-last-stable -) -RUN_PROGRAM( - ydb/tests/library/compatibility/downloader stable-24-4/release/ydbd ydbd-prelast-stable - OUT_NOAUTO ydbd-prelast-stable +PY_SRCS( + fixtures.py ) END() - -RECURSE(downloader)