[verifier] Adding ability to run checksum queries on test cluster.

spershin · spershin · commit 0700ff2b6372 · 2025-01-17T16:03:25.000-08:00
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForChecksum.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/annotation/ForChecksum.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.verifier.annotation;
+
+import javax.inject.Qualifier;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@Retention(RUNTIME)
+@Target({FIELD, PARAMETER, METHOD})
+@Qualifier
+public @interface ForChecksum
+{
+}
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/AbstractVerification.java
@@ -166,6 +166,11 @@ protected PrestoAction getHelperAction()
         return queryActions.getHelperAction();
     }
 
+    protected PrestoAction getChecksumAction()
+    {
+        return queryActions.getChecksumAction();
+    }
+
     protected boolean isControlEnabled()
     {
         return !skipControl || saveSnapshot;
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/framework/DataVerification.java
@@ -107,20 +107,20 @@ public DataMatchResult verify(
             ChecksumQueryContext controlChecksumQueryContext,
             ChecksumQueryContext testChecksumQueryContext)
     {
-        List<Column> testColumns = getColumns(getHelperAction(), typeManager, test.getObjectName());
+        List<Column> testColumns = getColumns(getChecksumAction(), typeManager, test.getObjectName());
         Query testChecksumQuery = checksumValidator.generateChecksumQuery(test.getObjectName(), testColumns, test.getPartitionsPredicate());
         testChecksumQueryContext.setChecksumQuery(formatSql(testChecksumQuery));
 
         List<Column> controlColumns = null;
         ChecksumResult controlChecksumResult = null;
 
         if (isControlEnabled()) {
-            controlColumns = getColumns(getHelperAction(), typeManager, control.getObjectName());
+            controlColumns = getColumns(getChecksumAction(), typeManager, control.getObjectName());
             Query controlChecksumQuery = checksumValidator.generateChecksumQuery(control.getObjectName(), controlColumns, control.getPartitionsPredicate());
             controlChecksumQueryContext.setChecksumQuery(formatSql(controlChecksumQuery));
 
             QueryResult<ChecksumResult> controlChecksum = callAndConsume(
-                    () -> getHelperAction().execute(controlChecksumQuery, CONTROL_CHECKSUM, ChecksumResult::fromResultSet),
+                    () -> getChecksumAction().execute(controlChecksumQuery, CONTROL_CHECKSUM, ChecksumResult::fromResultSet),
                     stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(controlChecksumQueryContext::setChecksumQueryId));
             controlChecksumResult = getOnlyElement(controlChecksum.getResults());
 
@@ -151,7 +151,7 @@ else if (QUERY_BANK_MODE.equals(runningMode)) {
         }
 
         QueryResult<ChecksumResult> testChecksum = callAndConsume(
-                () -> getHelperAction().execute(testChecksumQuery, TEST_CHECKSUM, ChecksumResult::fromResultSet),
+                () -> getChecksumAction().execute(testChecksumQuery, TEST_CHECKSUM, ChecksumResult::fromResultSet),
                 stats -> stats.getQueryStats().map(QueryStats::getQueryId).ifPresent(testChecksumQueryContext::setChecksumQueryId));
         ChecksumResult testChecksumResult = getOnlyElement(testChecksum.getResults());
 
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActions.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActions.java
@@ -23,12 +23,14 @@
 public class QueryActions
 {
     private final PrestoAction helperAction;
+    private final PrestoAction checksumAction;
     private final QueryAction controlAction;
     private final QueryAction testAction;
 
-    public QueryActions(PrestoAction helperAction, QueryAction controlAction, QueryAction testAction)
+    public QueryActions(PrestoAction helperAction, PrestoAction checksumAction, QueryAction controlAction, QueryAction testAction)
     {
         this.helperAction = requireNonNull(helperAction, "helperAction is null");
+        this.checksumAction = requireNonNull(helperAction, "checksumAction is null");
         this.controlAction = requireNonNull(controlAction, "controlAction is null");
         this.testAction = requireNonNull(testAction, "testAction is null");
     }
@@ -38,6 +40,11 @@ public PrestoAction getHelperAction()
         return helperAction;
     }
 
+    public PrestoAction getChecksumAction()
+    {
+        return checksumAction;
+    }
+
     public QueryAction getControlAction()
     {
         return controlAction;
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsConfig.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsConfig.java
@@ -26,6 +26,7 @@ public class QueryActionsConfig
     private String controlQueryActionType = JdbcPrestoAction.QUERY_ACTION_TYPE;
     private String testQueryActionType = JdbcPrestoAction.QUERY_ACTION_TYPE;
     private boolean runHelperQueriesOnControl = true;
+    private boolean runChecksumQueriesOnControl;
 
     private Duration metadataTimeout = new Duration(3, MINUTES);
     private Duration checksumTimeout = new Duration(30, MINUTES);
@@ -68,6 +69,18 @@ public QueryActionsConfig setRunHelperQueriesOnControl(boolean runHelperQueriesO
         return this;
     }
 
+    public boolean isRunChecksumQueriesOnControl()
+    {
+        return runChecksumQueriesOnControl;
+    }
+
+    @Config("run-checksum-queries-on-control")
+    public QueryActionsConfig setRunChecksumQueriesOnControl(boolean runChecksumQueriesOnControl)
+    {
+        this.runChecksumQueriesOnControl = runChecksumQueriesOnControl;
+        return this;
+    }
+
     @MinDuration("1s")
     public Duration getMetadataTimeout()
     {
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsModule.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsModule.java
@@ -14,6 +14,7 @@
 package com.facebook.presto.verifier.prestoaction;
 
 import com.facebook.airlift.configuration.AbstractConfigurationAwareModule;
+import com.facebook.presto.verifier.annotation.ForChecksum;
 import com.facebook.presto.verifier.annotation.ForControl;
 import com.facebook.presto.verifier.annotation.ForHelper;
 import com.facebook.presto.verifier.annotation.ForTest;
@@ -84,6 +85,14 @@ protected void setup(Binder binder)
             configBinder(binder).bindConfig(PrestoActionConfig.class, ForHelper.class, "helper");
             binder.bind(PrestoActionFactory.class).annotatedWith(ForHelper.class).toProvider(new JdbcPrestoActionFactoryProvider(ForHelper.class)).in(SINGLETON);
         }
+        if (config.isRunChecksumQueriesOnControl()) {
+            checkArgument(controlQueryActionType.equals(JdbcPrestoAction.QUERY_ACTION_TYPE), "Cannot run checksum queries on control cluster because it is not a presto-jdbc action");
+            binder.bind(PrestoActionFactory.class).annotatedWith(ForChecksum.class).toProvider(new JdbcPrestoActionFactoryProvider(ForControl.class)).in(SINGLETON);
+        }
+        else {
+            checkArgument(testQueryActionType.equals(JdbcPrestoAction.QUERY_ACTION_TYPE), "Cannot run checksum queries on test cluster because it is not a presto-jdbc action");
+            binder.bind(PrestoActionFactory.class).annotatedWith(ForChecksum.class).toProvider(new JdbcPrestoActionFactoryProvider(ForTest.class)).in(SINGLETON);
+        }
     }
 
     public static class JdbcPrestoActionFactoryProvider
diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsProvider.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/QueryActionsProvider.java
@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.verifier.prestoaction;
 
+import com.facebook.presto.verifier.annotation.ForChecksum;
 import com.facebook.presto.verifier.annotation.ForControl;
 import com.facebook.presto.verifier.annotation.ForHelper;
 import com.facebook.presto.verifier.annotation.ForTest;
@@ -26,16 +27,19 @@ public class QueryActionsProvider
         implements QueryActionsFactory
 {
     private final PrestoActionFactory helpActionFactory;
+    private final PrestoActionFactory checksumActionFactory;
     private final QueryActionFactory controlActionFactory;
     private final QueryActionFactory testActionFactory;
 
     @Inject
     public QueryActionsProvider(
             @ForHelper PrestoActionFactory helpActionFactory,
+            @ForChecksum PrestoActionFactory checksumActionFactory,
             @ForControl QueryActionFactory controlActionFactory,
             @ForTest QueryActionFactory testActionFactory)
     {
         this.helpActionFactory = requireNonNull(helpActionFactory, "helpActionFactory is null");
+        this.checksumActionFactory = requireNonNull(checksumActionFactory, "checksumActionFactory is null");
         this.controlActionFactory = requireNonNull(controlActionFactory, "controlActionFactory is null");
         this.testActionFactory = requireNonNull(testActionFactory, "testActionFactory is null");
     }
@@ -44,6 +48,7 @@ public QueryActions create(SourceQuery sourceQuery, VerificationContext verifica
     {
         return new QueryActions(
                 helpActionFactory.create(sourceQuery.getControlConfiguration(), verificationContext),
+                checksumActionFactory.create(sourceQuery.getTestConfiguration(), verificationContext),
                 controlActionFactory.create(sourceQuery.getControlConfiguration(), verificationContext),
                 testActionFactory.create(sourceQuery.getTestConfiguration(), verificationContext));
     }

Original file line number	Diff line number	Diff line change
`@@ -166,6 +166,11 @@ protected PrestoAction getHelperAction()`
`166`	`166`	`return queryActions.getHelperAction();`
`167`	`167`	`}`
`168`	`168`
	`169`	`+ protected PrestoAction getChecksumAction()`
	`170`	`+ {`
	`171`	`+ return queryActions.getChecksumAction();`
	`172`	`+ }`
	`173`	`+`
`169`	`174`	`protected boolean isControlEnabled()`
`170`	`175`	`{`
`171`	`176`	`return !skipControl \|\| saveSnapshot;`
Original file line number	Diff line number	Diff line change
`@@ -23,12 +23,14 @@`
`23`	`23`	`public class QueryActions`
`24`	`24`	`{`
`25`	`25`	`private final PrestoAction helperAction;`
	`26`	`+ private final PrestoAction checksumAction;`
`26`	`27`	`private final QueryAction controlAction;`
`27`	`28`	`private final QueryAction testAction;`
`28`	`29`
`29`		`- public QueryActions(PrestoAction helperAction, QueryAction controlAction, QueryAction testAction)`
	`30`	`+ public QueryActions(PrestoAction helperAction, PrestoAction checksumAction, QueryAction controlAction, QueryAction testAction)`
`30`	`31`	`{`
`31`	`32`	`this.helperAction = requireNonNull(helperAction, "helperAction is null");`
	`33`	`+ this.checksumAction = requireNonNull(helperAction, "checksumAction is null");`
`32`	`34`	`this.controlAction = requireNonNull(controlAction, "controlAction is null");`
`33`	`35`	`this.testAction = requireNonNull(testAction, "testAction is null");`
`34`	`36`	`}`
`@@ -38,6 +40,11 @@ public PrestoAction getHelperAction()`
`38`	`40`	`return helperAction;`
`39`	`41`	`}`
`40`	`42`
	`43`	`+ public PrestoAction getChecksumAction()`
	`44`	`+ {`
	`45`	`+ return checksumAction;`
	`46`	`+ }`
	`47`	`+`
`41`	`48`	`public QueryAction getControlAction()`
`42`	`49`	`{`
`43`	`50`	`return controlAction;`