Bug#34685358 (3/3) Integrate crund build in our cmake files

jdduncan · jdduncan · commit 64f6c788fff4 · 2025-02-03T20:52:41.000-08:00
This patch contains the new test driver runCrund.
Now it is possible to:

 * Use mtr --start-and-test crund to start a CRUND environment
 * Use runCrund, which by default will run ndbjtieAB and
   clusterjAB.
 * runCrund takes a list of properties and options on the
   command line which are passed to the C++ and Java crund
   utilities; for example:

     bin/runCrund nRuns=3 loads=NdbapiAB,NdbjtieAB,ClusterjAB
     bin/runCrund --show-properties

 * After running the benchmark, if the property nRuns &gt;= 3,
   runCrund will run the ResultProcessor to create a summarized
   CSV file, and then load this into the results table in the
   crunddb database. nRuns must be 3 or greater because the
   ResultProcessor treats the first two runs as warm-up runs
   and discards them.

   It should generally be possible to open the result CSV file
   in a spreadsheet application.

Miscellaneous changes in this patch include:

 * Remove the B_clearA operation from the crund AB workload.
   This was legal in NDB 6.x, but is no longer a legal
   operation due to the foreign key constraint violation.

 * Add a missing main() method to the S workloads in Java.

 * Change some shadowed names that trigger compiler warnings.

 * Add a --show-properties option to the crund programs.

Change-Id: I675ec2673dc9b9ced666488a7feff4a7e62a89ad
diff --git a/storage/ndb/test/crund/CMakeLists.txt b/storage/ndb/test/crund/CMakeLists.txt
@@ -26,11 +26,11 @@ INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/helpers)
 #
 ## runCrund
 ###
-# NDB_ADD_EXECUTABLE(runCrund runCrund.cpp NDBCLIENT)
-# TARGET_COMPILE_DEFINITIONS(runCrund PRIVATE
-#                           CMAKE_BINARY_DIR=\"${CMAKE_BINARY_DIR}\"
-#                           WITH_CLASSPATH=\"${WITH_CLASSPATH}\"
-#                          )
+NDB_ADD_EXECUTABLE(runCrund runCrund.cpp NDBCLIENT)
+TARGET_COMPILE_DEFINITIONS(runCrund PRIVATE
+                           CMAKE_BINARY_DIR=\"${CMAKE_BINARY_DIR}\"
+                           WITH_CLASSPATH=\"${WITH_CLASSPATH}\"
+                          )
 
 #
 ## C++ Crund Helpers
diff --git a/storage/ndb/test/crund/cpp/Driver.cpp b/storage/ndb/test/crund/cpp/Driver.cpp
@@ -123,15 +123,14 @@ void Driver::init() {
   loadProperties();
   initProperties();
   printProperties();
-  if (showPropsAndExit) return;
+  if (showPropsAndExit) exit(1);
 
   openLogFile();
   clearLogBuffers();
   initLoads();
 }
 
 void Driver::close() {
-  if (showPropsAndExit) return;
   closeLoads();
   clearLogBuffers();
   closeLogFile();
@@ -259,8 +258,6 @@ void Driver::addLoads() {
 
 void Driver::runLoads() {
   if (hasIgnoredSettings) {
-    if (showPropsAndExit) return;
-
     cout << endl
          << "++++++++++++  SOME SETTINGS IGNORED, SEE ABOVE  ++++++++++++"
          << endl;
diff --git a/storage/ndb/test/crund/cpp/unused-tws/NdbapiTwsDriver.cpp b/storage/ndb/test/crund/cpp/unused-tws/NdbapiTwsDriver.cpp
@@ -707,7 +707,7 @@ void NdbapiTwsDriver::initConnection() {
   cout << "creating cluster connection ..." << flush;
   assert(!mgmdConnect.empty());
   mgmd = new Ndb_cluster_connection(mgmdConnect.c_str());
-  mgmd->configure_tls(opt_tls_search_path, opt_mgm_tls);
+  //  mgmd->configure_tls(opt_tls_search_path, opt_mgm_tls);
   cout << " [ok: mgmd@" << mgmdConnect << "]" << endl;
 
   // connect to cluster management node (ndb_mgmd)
diff --git a/storage/ndb/test/crund/cpp/unused-tws/TwsDriver.cpp b/storage/ndb/test/crund/cpp/unused-tws/TwsDriver.cpp
@@ -146,7 +146,7 @@ void TwsDriver::runTests() {
 
   assert(nOpsStart <= nOpsEnd && nOpsScale > 1);
   for (int i = nOpsStart; i <= nOpsEnd; i *= nOpsScale) {
-    runLoads(i);
+    runLoads_(i);
   }
 
   cout << endl
@@ -157,7 +157,7 @@ void TwsDriver::runTests() {
   closeConnection();
 }
 
-void TwsDriver::runLoads(int nOps) {
+void TwsDriver::runLoads_(int nOps) {
   cout << endl
        << "------------------------------------------------------------"
        << endl;
diff --git a/storage/ndb/test/crund/cpp/unused-tws/TwsDriver.hpp b/storage/ndb/test/crund/cpp/unused-tws/TwsDriver.hpp
@@ -37,7 +37,6 @@ class TwsDriver : public Driver {
   static const char *toStr(XMode mode);
   bool renewConnection;
   bool renewOperations;
-  bool logSumOfOps;
   LockMode lockMode;
   int nOpsStart;
   int nOpsEnd;
@@ -59,7 +58,7 @@ class TwsDriver : public Driver {
   virtual void initOperations() = 0;
   virtual void closeOperations() = 0;
   virtual void runTests();
-  virtual void runLoads(int nOps);
+  virtual void runLoads_(int nOps);
   virtual void runOperations(int nOps);
   virtual void runInserts(XMode mode, int nOps) = 0;
   virtual void runLookups(XMode mode, int nOps) = 0;
diff --git a/storage/ndb/test/crund/helpers/hrt_stopwatch.h b/storage/ndb/test/crund/helpers/hrt_stopwatch.h
@@ -45,8 +45,8 @@ extern "C" {
  */
 typedef struct hrt_stopwatch {
   hrt_tstamp *tstamps;
-  unsigned int top;
-  unsigned int cap;
+  int top;
+  int cap;
 } hrt_stopwatch;
 
 /*
diff --git a/storage/ndb/test/crund/helpers/hrt_utils.h b/storage/ndb/test/crund/helpers/hrt_utils.h
@@ -29,6 +29,8 @@
  *
  */
 
+#include "ndb_config.h"
+
 #ifndef _utils_hrt_utils
 #define _utils_hrt_utils
 #ifdef __cplusplus
@@ -192,7 +194,7 @@ extern "C" {
 #error "unsupported HRT_REALTIME_METHOD: " HRT_REALTIME_METHOD
 #endif
 #else
-#if (defined(_POSIX_TIMERS) && (_POSIX_TIMERS + 0 >= 0))
+#if HAVE_CLOCK_GETTIME
 #define HRT_REALTIME_METHOD HRT_USE_CLOCK_GETTIME
 #else
 #define HRT_REALTIME_METHOD HRT_USE_GETTIMEOFDAY
@@ -206,8 +208,7 @@ extern "C" {
 #error "unsupported HRT_CPUTIME_METHOD: " HRT_CPUTIME_METHOD
 #endif
 #else
-#if (defined(_POSIX_TIMERS) && (_POSIX_TIMERS + 0 >= 0) && \
-     defined(_POSIX_CPUTIME) && (_POSIX_CPUTIME + 0 >= 0))
+#if HAVE_CLOCK_GETTIME
 #define HRT_CPUTIME_METHOD HRT_USE_CLOCK_GETTIME
 #else
 #define HRT_CPUTIME_METHOD HRT_USE_GETRUSAGE
diff --git a/storage/ndb/test/crund/java/ClusterjAB.java b/storage/ndb/test/crund/java/ClusterjAB.java
@@ -614,17 +614,6 @@ protected void check(int id, IB o) {
                     }
                 });
 
-            ops.add(
-                new WriteOp("B_clearA", xMode) {
-                    protected void write(int id) {
-                        // blind update
-                        final IB o = session.newInstance(IB.class);
-                        o.setId(id);
-                        final int aId = -1;
-                        o.setAid(aId);
-                        session.updatePersistent(o);
-                    }
-                });
 
             ops.add(
                 new WriteOp("B_del", xMode) {
diff --git a/storage/ndb/test/crund/java/ClusterjS.java b/storage/ndb/test/crund/java/ClusterjS.java
@@ -49,6 +49,16 @@ public ClusterjS(CrundDriver driver) {
         super(driver);
     }
 
+    static public void main(String[] args) {
+        System.out.println("ClusterjS.main()");
+        CrundDriver.parseArguments(args);
+        final CrundDriver driver = new CrundDriver();
+        final CrundSLoad load = new ClusterjS(driver);
+        driver.run();
+        System.out.println();
+        System.out.println("ClusterjS.main(): done.");
+    }
+
     // ----------------------------------------------------------------------
     // ClusterJ intializers/finalizers
     // ----------------------------------------------------------------------
diff --git a/storage/ndb/test/crund/java/Driver.java b/storage/ndb/test/crund/java/Driver.java
@@ -64,6 +64,7 @@ public abstract class Driver {
     // driver command-line arguments
     static private final List<String> propFileNames = new ArrayList<String>();
     static private String logFileName;
+    static protected boolean showPropsAndExit = false;
 
     // driver settings
     protected final Properties props = new Properties();
@@ -99,6 +100,7 @@ static protected void exitUsage() {
         out.println("    [-p <file name>]...    properties file name");
         out.println("    [-l <file name>]       log file name for results");
         out.println("    [-h|--help]            print usage message and exit");
+        out.println("    [--show-properties]    display properties and exit");
         out.println();
         System.exit(1); // return an error code
     }
@@ -121,6 +123,8 @@ static public void parseArguments(String[] args) {
                 logFileName = args[++i];
             } else if (arg.equals("-h") || arg.equals("--help")) {
                 exitUsage();
+            } else if (arg.equals("--show-properties")) {
+                showPropsAndExit = true;
             } else {
                 out.println("unknown option: " + arg);
                 exitUsage();
@@ -191,6 +195,8 @@ protected void init() throws Exception {
         loadProperties();
         initProperties();
         printProperties();
+        if(showPropsAndExit) System.exit(1);
+
         writeProperties();
         openLogFile();
         clearLogBuffers();
diff --git a/storage/ndb/test/crund/java/JdbcS.java b/storage/ndb/test/crund/java/JdbcS.java
@@ -35,6 +35,17 @@ This program is designed to work with certain software (including
 import com.mysql.cluster.crund.CrundDriver.XMode;
 
 class JdbcS extends CrundSLoad {
+
+    static public void main(String[] args) {
+        System.out.println("JdbcS.main()");
+        CrundDriver.parseArguments(args);
+        final CrundDriver driver = new CrundDriver();
+        final CrundSLoad load = new JdbcS(driver);
+        driver.run();
+        System.out.println();
+        System.out.println("JdbcS.main(): done.");
+    }
+
     // JDBC settings
     protected String jdbcDriver;
     protected String url;
diff --git a/storage/ndb/test/crund/java/NdbjtieAB.java b/storage/ndb/test/crund/java/NdbjtieAB.java
@@ -1236,15 +1236,6 @@ protected void check(int id) {
                     }
                 });
 
-            ops.add(
-                new UpdateOp("B_clearA", xMode, model.table_B) {
-                    public void setValues(int id) {
-                        setKeyB(op, id); // needs to be set first
-                        final int aid = -1;
-                        setAIdB(op, aid);
-                    }
-                });
-
             ops.add(
                 new DeleteOp("B_del", xMode, model.table_B) {
                     public void setValues(int id) {
diff --git a/storage/ndb/test/crund/java/NdbjtieS.java b/storage/ndb/test/crund/java/NdbjtieS.java
@@ -59,6 +59,17 @@ This program is designed to work with certain software (including
 import com.mysql.cluster.crund.CrundDriver.XMode;
 
 class NdbjtieS extends CrundSLoad {
+
+    static public void main(String[] args) {
+        System.out.println("NdbjtieS.main()");
+        CrundDriver.parseArguments(args);
+        final CrundDriver driver = new CrundDriver();
+        final CrundSLoad load = new NdbjtieS(driver);
+        driver.run();
+        System.out.println();
+        System.out.println("NdbjtieS.main(): done.");
+    }
+
     // NDB settings
     protected String mgmdConnect;
     protected String catalog;
diff --git a/storage/ndb/test/crund/java/ResultProcessor.java b/storage/ndb/test/crund/java/ResultProcessor.java
@@ -33,6 +33,10 @@ This program is designed to work with certain software (including
 import java.util.EnumSet;
 import java.text.ParseException;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.FileSystems;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileReader;
@@ -41,7 +45,6 @@ This program is designed to work with certain software (including
 import java.io.PrintWriter;
 import java.io.InputStream;
 
-
 /**
  * Processes the result files of the Crund benchmark.
  */
@@ -106,19 +109,22 @@ public void init() throws IOException {
         public void close() throws IOException {
             assert tmp != null;
             assert dst != null;
-            dst.flush();
-            final File to = new File(outFilePrefix + "." + suffix);
-            if (tmp.renameTo(to)) {
+            dst.close();
+            final Path to =
+                FileSystems.getDefault().getPath(outFilePrefix + "." + suffix);
+            try {
+                Files.move(tmp.toPath(), to);
                 out.println("wrote file: \t" + to);
-            } else {
+            }
+            catch(IOException ex) {
                 out.flush();
                 err.println("ERROR: failed to rename temp file:" + endl
                             + "    '" + tmp + "'" + endl
                             + " to output file:" + endl
                             + "    '" + to + "'");
+                throw ex;
             }
             tmp = null;
-            dst.close();
             dst = null;
         }
 
@@ -428,7 +434,7 @@ static protected void exitUsage() {
         err.println("usage: [options] <log file>...");
         err.println("    [-w <number>]       skip w warmup runs [w=0]");
         err.println("    [-d <number>]       mark ops with rsdev>d [d=10]");
-        err.println("    [-f txt,csv]        generate output formats [txt]");
+        err.println("    [-f txt,csv]        generate output formats [csv]");
         err.println("    [-o <file>]         out file prefix [./log_results]");
         err.println("    [-h|--help]         print usage message and exit");
         err.println();
@@ -439,7 +445,7 @@ static protected void exitUsage() {
     static public void parseArguments(String[] args) {
         String w = "0";
         String d = "10";
-        String f = "txt";
+        String f = "csv";
         String o = "./log_results";
         for (int i = 0; i < args.length; i++) {
             final String arg = args[i];
diff --git a/storage/ndb/test/crund/runCrund.cpp b/storage/ndb/test/crund/runCrund.cpp

Original file line number	Diff line number	Diff line change
`@@ -146,7 +146,7 @@ void TwsDriver::runTests() {`
`146`	`146`
`147`	`147`	`assert(nOpsStart <= nOpsEnd && nOpsScale > 1);`
`148`	`148`	`for (int i = nOpsStart; i <= nOpsEnd; i *= nOpsScale) {`
`149`		`- runLoads(i);`
	`149`	`+ runLoads_(i);`
`150`	`150`	`}`
`151`	`151`
`152`	`152`	`cout << endl`
`@@ -157,7 +157,7 @@ void TwsDriver::runTests() {`
`157`	`157`	`closeConnection();`
`158`	`158`	`}`
`159`	`159`
`160`		`-void TwsDriver::runLoads(int nOps) {`
	`160`	`+void TwsDriver::runLoads_(int nOps) {`
`161`	`161`	`cout << endl`
`162`	`162`	`<< "------------------------------------------------------------"`
`163`	`163`	`<< endl;`