database: Add vitess + mysql 8.0 to our development environment

.github/workflows/boulder-ci.yml

@@ -54,12 +54,19 @@ jobs:
           - "./t.sh --unit --enable-race-detection"
           - "./tn.sh --unit --enable-race-detection"
           - "./t.sh --start-py"
+          # Same cases but backed by Vitess + MySQL 8 instead of ProxySQL + MariaDB
+          - "./t.sh --use-vitess --integration"
+          - "./tn.sh --use-vitess --integration"
+          - "./t.sh --use-vitess --unit --enable-race-detection"
+          - "./tn.sh --use-vitess --unit --enable-race-detection"
+          - "./t.sh --use-vitess --start-py"
 
     env:
       # This sets the docker image tag for the boulder-tools repository to
       # use in tests. It will be set appropriately for each tag in the list
       # defined in the matrix.
       BOULDER_TOOLS_TAG: ${{ matrix.BOULDER_TOOLS_TAG }}
+      BOULDER_VTCOMBOSERVER_TAG: vitessv22.0.0_2025-11-03
 
     # Sequence of tasks that will be executed as part of the job.
     steps:

.gitignore

@@ -43,3 +43,11 @@ test/proxysql/*.log*
 
 # Coverage files
 test/coverage
+
+# DSN symlinks
+test/secrets/badkeyrevoker_dburl
+test/secrets/cert_checker_dburl
+test/secrets/incidents_dburl
+test/secrets/revoker_dburl
+test/secrets/sa_dburl
+test/secrets/sa_ro_dburl

docker-compose.yml

@@ -50,8 +50,9 @@ services:
       - 4001:4001 # ACMEv2
       - 4003:4003 # SFE
     depends_on:
-      - bmysql
+      - bmariadb
       - bproxysql
+      - bvitess
       - bredis_1
       - bredis_2
       - bconsul
@@ -74,12 +75,12 @@ services:
       # with a "docker compose up bsetup".
       - setup
 
-  bmysql:
+  bmariadb:
     image: mariadb:10.11.13
     networks:
       bouldernet:
         aliases:
-          - boulder-mysql
+          - boulder-mariadb
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
     # Send slow queries to a table so we can check for them in the
@@ -101,7 +102,7 @@ services:
     volumes:
       - ./test/:/test/:cached
     depends_on:
-      - bmysql
+      - bmariadb
     networks:
       bouldernet:
         aliases:
@@ -144,6 +145,21 @@ services:
     networks:
       - bouldernet
 
+  bvitess:
+    # The `letsencrypt/boulder-vtcomboserver:latest` tag is automatically built
+    # in local dev environments. In CI a specific BOULDER_VTCOMBOSERVER_TAG is
+    # passed, and it is pulled with `docker compose pull`.
+    image: letsencrypt/boulder-vtcomboserver:${BOULDER_VTCOMBOSERVER_TAG:-latest}
+    environment:
+      # By specifying KEYSPACES vttestserver will create the corresponding
+      # databases on startup.
+      KEYSPACES: boulder_sa_test,boulder_sa_integration,incidents_sa_test,incidents_sa_integration
+      NUM_SHARDS: 1,1,1,1
+    networks:
+      bouldernet:
+        aliases:
+          - boulder-vitess
+
 networks:
   # This network represents the data-center internal network. It is used for
   # boulder services and their infrastructure, such as consul, mariadb, and

sa/db-next/dbconfig.mysql8.yml

@@ -0,0 +1 @@
+../db/dbconfig.mysql8.yml

sa/db/dbconfig.mysql8.yml

@@ -0,0 +1,20 @@
+# https://github.com/rubenv/sql-migrate#readme
+boulder_sa_test:
+  dialect: mysql
+  datasource: root@tcp(boulder-vitess:33577)/boulder_sa_test?parseTime=true
+  dir: boulder_sa
+
+boulder_sa_integration:
+  dialect: mysql
+  datasource: root@tcp(boulder-vitess:33577)/boulder_sa_integration?parseTime=true
+  dir: boulder_sa
+
+incidents_sa_test:
+  dialect: mysql
+  datasource: root@tcp(boulder-vitess:33577)/incidents_sa_test?parseTime=true
+  dir: incidents_sa
+
+incidents_sa_integration:
+  dialect: mysql
+  datasource: root@tcp(boulder-vitess:33577)/incidents_sa_integration?parseTime=true
+  dir: incidents_sa

test.sh

@@ -12,7 +12,7 @@ fi
 # Defaults
 #
 export RACE="false"
-export DB_ADDR="boulder-proxysql:6033"
+export USE_VITESS="false"
 STAGE="starting"
 STATUS="FAILURE"
 RUN=()
@@ -22,6 +22,14 @@ INTEGRATION_FLAGS=()
 FILTER=()
 COVERAGE="false"
 COVERAGE_DIR="test/coverage/$(date +%Y-%m-%d_%H-%M-%S)"
+DB_URL_FILES=(
+  badkeyrevoker_dburl
+  cert_checker_dburl
+  incidents_dburl
+  revoker_dburl
+  sa_dburl
+  sa_ro_dburl
+)
 
 #
 # Cleanup Functions
@@ -79,6 +87,23 @@ function run_and_expect_silence() {
   rm "${result_file}"
 }
 
+configure_database_endpoints() {
+  dburl_target_dir="proxysql"
+  export DB_ADDR="boulder-proxysql:6033"
+
+  if [[ "${USE_VITESS}" == "true" ]]
+  then
+    dburl_target_dir="vitess"
+    export DB_ADDR="boulder-vitess:33577"
+  fi
+
+  # Configure DBURL symlinks
+  rm -f test/secrets/*_dburl || true
+  for file in ${DB_URL_FILES:+${DB_URL_FILES[@]+"${DB_URL_FILES[@]}"}}
+  do
+    ln -sf "dburls/${dburl_target_dir}/${file}" "test/secrets/${file}"
+  done
+}
 #
 # Testing Helpers
 #
@@ -122,11 +147,12 @@ With no options passed, runs standard battery of tests (lint, unit, and integrat
                                           Example:
                                            TestGenerateValidity/TestWFECORS
     -h, --help                            Shows this help message
+    -b  --use-vitess                      Run tests against Vitess + MySQL 8.0 database
 
 EOM
 )"
 
-while getopts luvwecisgnhd:p:f:-: OPT; do
+while getopts luvwecisgnhbd:p:f:-: OPT; do
   if [ "$OPT" = - ]; then     # long option: reformulate OPT and OPTARG
     OPT="${OPTARG%%=*}"       # extract long option name
     OPTARG="${OPTARG#$OPT}"   # extract long option argument (may be empty)
@@ -146,13 +172,17 @@ while getopts luvwecisgnhd:p:f:-: OPT; do
     n | config-next )                BOULDER_CONFIG_DIR="test/config-next" ;;
     c | coverage )                   COVERAGE="true" ;;
     d | coverage-dir )               check_arg; COVERAGE_DIR="${OPTARG}" ;;
+    b | use-vitess )                 USE_VITESS="true" ;;
     h | help )                       print_usage_exit ;;
     ??* )                            exit_msg "Illegal option --$OPT" ;;  # bad long option
     ? )                              exit 2 ;;  # bad short option (error reported via getopts)
   esac
 done
 shift $((OPTIND-1)) # remove parsed options and args from $@ list
 
+# Defaults to MariaDB unless USE_VITESS is true.
+configure_database_endpoints
+
 # The list of segments to run. Order doesn't matter.
 if [ -z "${RUN[@]+x}" ]
 then
@@ -207,6 +237,7 @@ settings="$(cat -- <<-EOM
     FILTER:             ${FILTER[@]}
     COVERAGE:           $COVERAGE
     COVERAGE_DIR:       $COVERAGE_DIR
+    USE_VITESS:         $USE_VITESS
 EOM
 )"
 

test/create_db.sh

@@ -44,28 +44,41 @@ function create_empty_db() {
 dbconn="-u root"
 if [[ $MYSQL_CONTAINER ]]
 then
-	dbconn="-u root -h boulder-mysql --port 3306"
+  dbconn="-u root -h ${DB_HOST} --port ${DB_PORT}"
 fi
 
-# MariaDB sets the default binlog_format to STATEMENT,
-# which causes warnings that fail tests. Instead set it
-# to the format we use in production, MIXED.
-mysql ${dbconn} -e "SET GLOBAL binlog_format = 'MIXED';"
+if ! mysql ${dbconn} -e "select 1" >/dev/null 2>&1; then
+  exit_err "unable to connect to ${DB_HOST}:${DB_PORT}"
+fi
 
-# MariaDB sets the default @@max_connections value to 100. The SA alone is
-# configured to use up to 100 connections. We increase the max connections here
-# to give headroom for other components.
-mysql ${dbconn} -e "SET GLOBAL max_connections = 500;"
+if [[ ${SKIP_CREATE} -eq 0 ]]
+then
+  # MariaDB sets the default binlog_format to STATEMENT,
+  # which causes warnings that fail tests. Instead set it
+  # to the format we use in production, MIXED.
+  mysql ${dbconn} -e "SET GLOBAL binlog_format = 'MIXED';"
+
+  # MariaDB sets the default @@max_connections value to 100. The SA alone is
+  # configured to use up to 100 connections. We increase the max connections here
+  # to give headroom for other components.
+  mysql ${dbconn} -e "SET GLOBAL max_connections = 500;"
+fi
 
 for db in $DBS; do
   for env in $ENVS; do
     dbname="${db}_${env}"
     print_heading "${dbname}"
-    if mysql ${dbconn} -e 'show databases;' | grep "${dbname}" > /dev/null; then
-      echo "Already exists - skipping create"
+    if [[ ${SKIP_CREATE} -eq 0 ]]
+    then
+      if mysql ${dbconn} -e 'show databases;' | grep -q "${dbname}"
+      then
+        echo "Already exists - skipping create"
+      else
+        echo "Doesn't exist - creating"
+        create_empty_db "${dbname}" "${dbconn}"
+      fi
     else
-      echo "Doesn't exist - creating"
-      create_empty_db "${dbname}" "${dbconn}"
+      echo "Skipping database create for ${dbname}"
     fi
 
     if [[ "${BOULDER_CONFIG_DIR}" == "test/config-next" ]]
@@ -78,27 +91,32 @@ for db in $DBS; do
     # sql-migrate will default to ./dbconfig.yml and treat all configured dirs
     # as relative.
     cd "${dbpath}"
-    r=`sql-migrate up -env="${dbname}" | xargs -0 echo`
+    r=`sql-migrate up -config="${DB_CONFIG_FILE}" -env="${dbname}" | xargs -0 echo`
     if [[ "${r}" == "Migration failed"* ]]
     then
       echo "Migration failed - dropping and recreating"
       create_empty_db "${dbname}" "${dbconn}"
-      sql-migrate up -env="${dbname}" || exit_err "Migration failed after dropping and recreating"
+      sql-migrate up -config="${DB_CONFIG_FILE}" -env="${dbname}" || exit_err "Migration failed after dropping and recreating"
     else
       echo "${r}"
     fi
 
     USERS_SQL="../db-users/${db}.sql"
-    if [[ ${MYSQL_CONTAINER} ]]
+    if [[ ${SKIP_USERS} -eq 1 ]]
     then
-      sed -e "s/'localhost'/'%'/g" < ${USERS_SQL} | \
-        mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}"
+      echo "Skipping user grants for ${dbname}"
     else
-      sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \
-        mysql ${dbconn} -D "${dbname}" -f < $USERS_SQL || exit_err "Unable to add users from ${USERS_SQL}"
+      if [[ $MYSQL_CONTAINER ]]
+      then
+        sed -e "s/'localhost'/'%'/g" < "${USERS_SQL}" | \
+          mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}"
+      else
+        sed -e "s/'localhost'/'127.%'/g" < "${USERS_SQL}" | \
+          mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}"
+      fi
+      echo "Added users from ${USERS_SQL}"
     fi
-    echo "Added users from ${USERS_SQL}"
-
+
     # return to the root directory
     cd "${root_dir}"
   done

test/entrypoint.sh

@@ -10,17 +10,32 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 rm -f /var/run/rsyslogd.pid
 rsyslogd
 
-# make sure we can reach the mysqldb.
-./test/wait-for-it.sh boulder-mysql 3306
+# make sure we can reach mariadb and proxysql
+./test/wait-for-it.sh boulder-mariadb 3306
+./test/wait-for-it.sh boulder-proxysql 6033
 
-# make sure we can reach the proxysql.
-./test/wait-for-it.sh bproxysql 6032
+# make sure we can reach vitess
+./test/wait-for-it.sh boulder-vitess 33577
 
 # make sure we can reach pkilint
 ./test/wait-for-it.sh bpkimetal 8080
 
-# create the database
-MYSQL_CONTAINER=1 $DIR/create_db.sh
+# create the databases
+MYSQL_CONTAINER=1 \
+DB_HOST="boulder-mariadb" \
+DB_PORT=3306 \
+DB_CONFIG_FILE="${DIR}/../sa/db/dbconfig.mariadb.yml" \
+SKIP_CREATE=0 \
+SKIP_USERS=0 \
+"$DIR/create_db.sh"
+
+MYSQL_CONTAINER=1 \
+DB_HOST="boulder-vitess" \
+DB_PORT=33577 \
+DB_CONFIG_FILE="${DIR}/../sa/db/dbconfig.mysql8.yml" \
+SKIP_CREATE=1 \
+SKIP_USERS=1 \
+"$DIR/create_db.sh"
 
 if [[ $# -eq 0 ]]; then
     exec python3 ./start.py