Extended test_046 to check clickhouse_operator_chi metric.

Altinity · Jan 12, 2025 · c21f884 · c21f884
1 parent 6369826
commit c21f884
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 21 deletions.
diff --git a/tests/e2e/manifests/chi/test-028-replication.yaml b/tests/e2e/manifests/chi/test-028-replication.yaml
@@ -21,8 +21,5 @@ spec:
     clusters:
       - name: default
         layout:
-          shardsCount: 2
+          shardsCount: 1
           replicasCount: 2
-    profiles:
-      default/database_atomic_wait_for_drop_and_detach_synchronously: 1
-      default/allow_experimental_live_view: 1
diff --git a/...manifests/chopconf/test-045-chopconf.yaml → ...e/manifests/chopconf/no-wait-queries.yaml b/...manifests/chopconf/test-045-chopconf.yaml → ...e/manifests/chopconf/no-wait-queries.yaml
@@ -1,7 +1,7 @@
 apiVersion: "clickhouse.altinity.com/v1"
 kind: "ClickHouseOperatorConfiguration"
 metadata:
-  name: "test-045-chopconf"
+  name: "no-wait-queries"
 spec:
   reconcile:
     host:

diff --git a/tests/e2e/steps.py b/tests/e2e/steps.py
@@ -188,4 +188,4 @@ def check_metrics_monitoring(
                 with Then("Not ready. Wait for " + str(i * 5) + " seconds"):
                     time.sleep(i * 5)
 
-        assert expected_pattern_found, error()
+        assert expected_pattern_found, error(out)
diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py
@@ -3029,7 +3029,6 @@ def test_027(self):
 @Requirements(RQ_SRS_026_ClickHouseOperator_Managing_RestartingOperator("1.0"))
 def test_028(self):
     create_shell_namespace_clickhouse_template()
-
     util.require_keeper(keeper_type=self.context.keeper_type)
 
     manifest = "manifests/chi/test-028-replication.yaml"
@@ -3043,17 +3042,17 @@ def test_028(self):
                 "manifests/chit/tpl-persistent-volume-100Mi.yaml",
             },
             "object_counts": {
-                "statefulset": 4,
-                "pod": 4,
-                "service": 5,
+                "statefulset": 2,
+                "pod": 2,
+                "service": 3,
             },
             "do_not_delete": 1,
         },
     )
 
-    sql = """SET skip_unavailable_shards=1; SYSTEM DROP DNS CACHE; SELECT getMacro('replica') AS replica, uptime() AS uptime,
+    sql = """SELECT getMacro('replica') AS replica, uptime() AS uptime,
      (SELECT count() FROM system.clusters WHERE cluster='all-sharded') AS total_hosts,
-     (SELECT count() online_hosts FROM cluster('all-sharded', system.one) ) AS online_hosts
+     (SELECT count() online_hosts FROM cluster('all-sharded', system.one) settings skip_unavailable_shards=1 ) AS online_hosts
      FORMAT JSONEachRow"""
     note("Before restart")
     out = clickhouse.query_with_error(chi, sql)
@@ -3085,8 +3084,8 @@ def test_028(self):
                     ch2 = clickhouse.query_with_error(
                         chi,
                         sql,
-                        pod="chi-test-028-replication-default-1-0-0",
-                        host="chi-test-028-replication-default-1-0",
+                        pod="chi-test-028-replication-default-0-1-0",
+                        host="chi-test-028-replication-default-0-1",
                         advanced_params="--connect_timeout=1 --send_timeout=10 --receive_timeout=10",
                     )
 
@@ -3124,16 +3123,16 @@ def test_028(self):
         with Then("Clear RollingUpdate restart policy"):
             cmd = f"patch chi {chi} --type='json' --patch='[{{\"op\":\"remove\",\"path\":\"/spec/restart\"}}]'"
             kubectl.launch(cmd)
-            time.sleep(15)
+            kubectl.wait_chi_status(chi, "InProgress")
             kubectl.wait_chi_status(chi, "Completed")
 
         with Then("Restart operator. CHI should not be restarted"):
             check_operator_restart(
                 chi=chi,
                 wait_objects={
-                    "statefulset": 4,
-                    "pod": 4,
-                    "service": 5,
+                    "statefulset": 2,
+                    "pod": 2,
+                    "service": 3,
                 },
                 pod=f"chi-{chi}-default-0-0-0",
             )
@@ -3148,12 +3147,13 @@ def test_028(self):
     with When("Stop installation"):
         cmd = f'patch chi {chi} --type=\'json\' --patch=\'[{{"op":"add","path":"/spec/stop","value":"yes"}}]\''
         kubectl.launch(cmd)
+        kubectl.wait_chi_status(chi, "InProgress")
         kubectl.wait_chi_status(chi, "Completed")
         with Then("Stateful sets should be there but no running pods"):
             kubectl.wait_objects(chi, {
-                "statefulset": 4,
+                "statefulset": 2,
                 "pod": 0,
-                "service": 4,
+                "service": 2,
             })
 
     with Finally("I clean up"):
@@ -4482,7 +4482,7 @@ def test_045_2(self):
     create_shell_namespace_clickhouse_template()
 
     with Given("I set spec.reconcile.host.wait.queries property"):
-        util.apply_operator_config("manifests/chopconf/test-045-chopconf.yaml")
+        util.apply_operator_config("manifests/chopconf/no-wait-queries.yaml")
 
     test_045(manifest=f"manifests/chi/test-045-2-wait-query-finish.yaml")
 
@@ -4492,6 +4492,9 @@ def test_045_2(self):
 def test_046(self):
     """Check that clickhouse-operator creates metrics for reconcile and other clickhouse-operator events."""
     create_shell_namespace_clickhouse_template()
+    with Given("I change operator statefullSet timeout"):
+        util.apply_operator_config("manifests/chopconf/low-timeout.yaml")
+
     cluster = "default"
     manifest = f"manifests/chi/test-046-0-clickhouse-operator-metrics.yaml"
     chi = yaml_manifest.get_name(util.get_full_path(manifest))
@@ -4517,8 +4520,15 @@ def check_metrics(metric_names):
                     container="clickhouse-operator",
                     port="9999",
                     expect_pattern=metric_name,
+                    max_retries=3
                 )
 
+    with Then(f"Check clickhouse-operator exposes clickhouse_operator_chi metrics"):
+        check_metrics([
+            "clickhouse_operator_chi{.*chi=\"test-046-operator-metrics\".*} 1",
+        ])
+
+
     with Then(f"Check clickhouse-operator exposes clickhouse_operator_chi_reconciles_* metrics"):
         check_metrics([
             "clickhouse_operator_chi_reconciles_started{.*chi=\"test-046-operator-metrics\".*} 1",
@@ -4593,6 +4603,17 @@ def check_metrics(metric_names):
             "clickhouse_operator_host_reconciles_timings.*chi=\"test-046-operator-metrics\".*",
             ])
 
+    with Then("Stop CHI"):
+        cmd = f'patch chi {chi} --type=\'json\' --patch=\'[{{"op":"add","path":"/spec/stop","value":"yes"}}]\''
+        kubectl.launch(cmd)
+        kubectl.wait_chi_status(chi, "InProgress")
+        kubectl.wait_chi_status(chi, "Completed")
+
+    with Then(f"Check clickhouse-operator exposes clickhouse_operator_chi metric for stopped chi"):
+        check_metrics([
+            "clickhouse_operator_chi{.*chi=\"test-046-operator-metrics\".*} 1",
+        ])
+
     with Finally("I clean up"):
         delete_test_namespace()