diff --git a/.github/workflows/boulder-ci.yml b/.github/workflows/boulder-ci.yml index 5513cd54cd6..17ecdf26fdf 100644 --- a/.github/workflows/boulder-ci.yml +++ b/.github/workflows/boulder-ci.yml @@ -54,12 +54,19 @@ jobs: - "./t.sh --unit --enable-race-detection" - "./tn.sh --unit --enable-race-detection" - "./t.sh --start-py" + # Same cases but backed by Vitess + MySQL 8 instead of ProxySQL + MariaDB + - "./t.sh --use-vitess --integration" + - "./tn.sh --use-vitess --integration" + - "./t.sh --use-vitess --unit --enable-race-detection" + - "./tn.sh --use-vitess --unit --enable-race-detection" + - "./t.sh --use-vitess --start-py" env: # This sets the docker image tag for the boulder-tools repository to # use in tests. It will be set appropriately for each tag in the list # defined in the matrix. BOULDER_TOOLS_TAG: ${{ matrix.BOULDER_TOOLS_TAG }} + BOULDER_VTCOMBOSERVER_TAG: vitessv22.0.0_2025-11-03 # Sequence of tasks that will be executed as part of the job. steps: diff --git a/.gitignore b/.gitignore index 769b54767cf..5e1426c919a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,11 @@ test/proxysql/*.log* # Coverage files test/coverage + +# DSN symlinks +test/secrets/badkeyrevoker_dburl +test/secrets/cert_checker_dburl +test/secrets/incidents_dburl +test/secrets/revoker_dburl +test/secrets/sa_dburl +test/secrets/sa_ro_dburl diff --git a/docker-compose.yml b/docker-compose.yml index 39802cd8b58..f21c9322521 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,8 +50,9 @@ services: - 4001:4001 # ACMEv2 - 4003:4003 # SFE depends_on: - - bmysql + - bmariadb - bproxysql + - bvitess - bredis_1 - bredis_2 - bconsul @@ -74,12 +75,12 @@ services: # with a "docker compose up bsetup". - setup - bmysql: + bmariadb: image: mariadb:10.11.13 networks: bouldernet: aliases: - - boulder-mysql + - boulder-mariadb environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" # Send slow queries to a table so we can check for them in the @@ -101,7 +102,7 @@ services: volumes: - ./test/:/test/:cached depends_on: - - bmysql + - bmariadb networks: bouldernet: aliases: @@ -144,6 +145,21 @@ services: networks: - bouldernet + bvitess: + # The `letsencrypt/boulder-vtcomboserver:latest` tag is automatically built + # in local dev environments. In CI a specific BOULDER_VTCOMBOSERVER_TAG is + # passed, and it is pulled with `docker compose pull`. + image: letsencrypt/boulder-vtcomboserver:${BOULDER_VTCOMBOSERVER_TAG:-latest} + environment: + # By specifying KEYSPACES vttestserver will create the corresponding + # databases on startup. + KEYSPACES: boulder_sa_test,boulder_sa_integration,incidents_sa_test,incidents_sa_integration + NUM_SHARDS: 1,1,1,1 + networks: + bouldernet: + aliases: + - boulder-vitess + networks: # This network represents the data-center internal network. It is used for # boulder services and their infrastructure, such as consul, mariadb, and diff --git a/sa/db-next/dbconfig.mysql8.yml b/sa/db-next/dbconfig.mysql8.yml new file mode 120000 index 00000000000..bc7d1365f7f --- /dev/null +++ b/sa/db-next/dbconfig.mysql8.yml @@ -0,0 +1 @@ +../db/dbconfig.mysql8.yml \ No newline at end of file diff --git a/sa/db-next/dbconfig.yml b/sa/db-next/dbconfig.yml deleted file mode 120000 index cb034f30bd7..00000000000 --- a/sa/db-next/dbconfig.yml +++ /dev/null @@ -1 +0,0 @@ -../db/dbconfig.yml \ No newline at end of file diff --git a/sa/db/dbconfig.yml b/sa/db/dbconfig.mariadb.yml similarity index 100% rename from sa/db/dbconfig.yml rename to sa/db/dbconfig.mariadb.yml diff --git a/sa/db/dbconfig.mysql8.yml b/sa/db/dbconfig.mysql8.yml new file mode 100644 index 00000000000..6f9c9bf6578 --- /dev/null +++ b/sa/db/dbconfig.mysql8.yml @@ -0,0 +1,20 @@ +# https://github.com/rubenv/sql-migrate#readme +boulder_sa_test: + dialect: mysql + datasource: root@tcp(boulder-vitess:33577)/boulder_sa_test?parseTime=true + dir: boulder_sa + +boulder_sa_integration: + dialect: mysql + datasource: root@tcp(boulder-vitess:33577)/boulder_sa_integration?parseTime=true + dir: boulder_sa + +incidents_sa_test: + dialect: mysql + datasource: root@tcp(boulder-vitess:33577)/incidents_sa_test?parseTime=true + dir: incidents_sa + +incidents_sa_integration: + dialect: mysql + datasource: root@tcp(boulder-vitess:33577)/incidents_sa_integration?parseTime=true + dir: incidents_sa diff --git a/test.sh b/test.sh index 2f55d25e97f..edc8bbaf4b6 100755 --- a/test.sh +++ b/test.sh @@ -12,7 +12,7 @@ fi # Defaults # export RACE="false" -export DB_ADDR="boulder-proxysql:6033" +export USE_VITESS="false" STAGE="starting" STATUS="FAILURE" RUN=() @@ -22,6 +22,14 @@ INTEGRATION_FLAGS=() FILTER=() COVERAGE="false" COVERAGE_DIR="test/coverage/$(date +%Y-%m-%d_%H-%M-%S)" +DB_URL_FILES=( + badkeyrevoker_dburl + cert_checker_dburl + incidents_dburl + revoker_dburl + sa_dburl + sa_ro_dburl +) # # Cleanup Functions @@ -79,6 +87,23 @@ function run_and_expect_silence() { rm "${result_file}" } +configure_database_endpoints() { + dburl_target_dir="proxysql" + export DB_ADDR="boulder-proxysql:6033" + + if [[ "${USE_VITESS}" == "true" ]] + then + dburl_target_dir="vitess" + export DB_ADDR="boulder-vitess:33577" + fi + + # Configure DBURL symlinks + rm -f test/secrets/*_dburl || true + for file in ${DB_URL_FILES:+${DB_URL_FILES[@]+"${DB_URL_FILES[@]}"}} + do + ln -sf "dburls/${dburl_target_dir}/${file}" "test/secrets/${file}" + done +} # # Testing Helpers # @@ -122,11 +147,12 @@ With no options passed, runs standard battery of tests (lint, unit, and integrat Example: TestGenerateValidity/TestWFECORS -h, --help Shows this help message + -b --use-vitess Run tests against Vitess + MySQL 8.0 database EOM )" -while getopts luvwecisgnhd:p:f:-: OPT; do +while getopts luvwecisgnhbd:p:f:-: OPT; do if [ "$OPT" = - ]; then # long option: reformulate OPT and OPTARG OPT="${OPTARG%%=*}" # extract long option name OPTARG="${OPTARG#$OPT}" # extract long option argument (may be empty) @@ -146,6 +172,7 @@ while getopts luvwecisgnhd:p:f:-: OPT; do n | config-next ) BOULDER_CONFIG_DIR="test/config-next" ;; c | coverage ) COVERAGE="true" ;; d | coverage-dir ) check_arg; COVERAGE_DIR="${OPTARG}" ;; + b | use-vitess ) USE_VITESS="true" ;; h | help ) print_usage_exit ;; ??* ) exit_msg "Illegal option --$OPT" ;; # bad long option ? ) exit 2 ;; # bad short option (error reported via getopts) @@ -153,6 +180,9 @@ while getopts luvwecisgnhd:p:f:-: OPT; do done shift $((OPTIND-1)) # remove parsed options and args from $@ list +# Defaults to MariaDB unless USE_VITESS is true. +configure_database_endpoints + # The list of segments to run. Order doesn't matter. if [ -z "${RUN[@]+x}" ] then @@ -207,6 +237,7 @@ settings="$(cat -- <<-EOM FILTER: ${FILTER[@]} COVERAGE: $COVERAGE COVERAGE_DIR: $COVERAGE_DIR + USE_VITESS: $USE_VITESS EOM )" diff --git a/test/create_db.sh b/test/create_db.sh index d43a9ace116..d485b243601 100755 --- a/test/create_db.sh +++ b/test/create_db.sh @@ -44,28 +44,41 @@ function create_empty_db() { dbconn="-u root" if [[ $MYSQL_CONTAINER ]] then - dbconn="-u root -h boulder-mysql --port 3306" + dbconn="-u root -h ${DB_HOST} --port ${DB_PORT}" fi -# MariaDB sets the default binlog_format to STATEMENT, -# which causes warnings that fail tests. Instead set it -# to the format we use in production, MIXED. -mysql ${dbconn} -e "SET GLOBAL binlog_format = 'MIXED';" +if ! mysql ${dbconn} -e "select 1" >/dev/null 2>&1; then + exit_err "unable to connect to ${DB_HOST}:${DB_PORT}" +fi -# MariaDB sets the default @@max_connections value to 100. The SA alone is -# configured to use up to 100 connections. We increase the max connections here -# to give headroom for other components. -mysql ${dbconn} -e "SET GLOBAL max_connections = 500;" +if [[ ${SKIP_CREATE} -eq 0 ]] +then + # MariaDB sets the default binlog_format to STATEMENT, + # which causes warnings that fail tests. Instead set it + # to the format we use in production, MIXED. + mysql ${dbconn} -e "SET GLOBAL binlog_format = 'MIXED';" + + # MariaDB sets the default @@max_connections value to 100. The SA alone is + # configured to use up to 100 connections. We increase the max connections here + # to give headroom for other components. + mysql ${dbconn} -e "SET GLOBAL max_connections = 500;" +fi for db in $DBS; do for env in $ENVS; do dbname="${db}_${env}" print_heading "${dbname}" - if mysql ${dbconn} -e 'show databases;' | grep "${dbname}" > /dev/null; then - echo "Already exists - skipping create" + if [[ ${SKIP_CREATE} -eq 0 ]] + then + if mysql ${dbconn} -e 'show databases;' | grep -q "${dbname}" + then + echo "Already exists - skipping create" + else + echo "Doesn't exist - creating" + create_empty_db "${dbname}" "${dbconn}" + fi else - echo "Doesn't exist - creating" - create_empty_db "${dbname}" "${dbconn}" + echo "Skipping database create for ${dbname}" fi if [[ "${BOULDER_CONFIG_DIR}" == "test/config-next" ]] @@ -78,27 +91,32 @@ for db in $DBS; do # sql-migrate will default to ./dbconfig.yml and treat all configured dirs # as relative. cd "${dbpath}" - r=`sql-migrate up -env="${dbname}" | xargs -0 echo` + r=`sql-migrate up -config="${DB_CONFIG_FILE}" -env="${dbname}" | xargs -0 echo` if [[ "${r}" == "Migration failed"* ]] then echo "Migration failed - dropping and recreating" create_empty_db "${dbname}" "${dbconn}" - sql-migrate up -env="${dbname}" || exit_err "Migration failed after dropping and recreating" + sql-migrate up -config="${DB_CONFIG_FILE}" -env="${dbname}" || exit_err "Migration failed after dropping and recreating" else echo "${r}" fi USERS_SQL="../db-users/${db}.sql" - if [[ ${MYSQL_CONTAINER} ]] + if [[ ${SKIP_USERS} -eq 1 ]] then - sed -e "s/'localhost'/'%'/g" < ${USERS_SQL} | \ - mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}" + echo "Skipping user grants for ${dbname}" else - sed -e "s/'localhost'/'127.%'/g" < $USERS_SQL | \ - mysql ${dbconn} -D "${dbname}" -f < $USERS_SQL || exit_err "Unable to add users from ${USERS_SQL}" + if [[ $MYSQL_CONTAINER ]] + then + sed -e "s/'localhost'/'%'/g" < "${USERS_SQL}" | \ + mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}" + else + sed -e "s/'localhost'/'127.%'/g" < "${USERS_SQL}" | \ + mysql ${dbconn} -D "${dbname}" -f || exit_err "Unable to add users from ${USERS_SQL}" + fi + echo "Added users from ${USERS_SQL}" fi - echo "Added users from ${USERS_SQL}" - + # return to the root directory cd "${root_dir}" done diff --git a/test/entrypoint.sh b/test/entrypoint.sh index 1d8c363c5d0..331c0c731df 100755 --- a/test/entrypoint.sh +++ b/test/entrypoint.sh @@ -10,17 +10,32 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" rm -f /var/run/rsyslogd.pid rsyslogd -# make sure we can reach the mysqldb. -./test/wait-for-it.sh boulder-mysql 3306 +# make sure we can reach mariadb and proxysql +./test/wait-for-it.sh boulder-mariadb 3306 +./test/wait-for-it.sh boulder-proxysql 6033 -# make sure we can reach the proxysql. -./test/wait-for-it.sh bproxysql 6032 +# make sure we can reach vitess +./test/wait-for-it.sh boulder-vitess 33577 # make sure we can reach pkilint ./test/wait-for-it.sh bpkimetal 8080 -# create the database -MYSQL_CONTAINER=1 $DIR/create_db.sh +# create the databases +MYSQL_CONTAINER=1 \ +DB_HOST="boulder-mariadb" \ +DB_PORT=3306 \ +DB_CONFIG_FILE="${DIR}/../sa/db/dbconfig.mariadb.yml" \ +SKIP_CREATE=0 \ +SKIP_USERS=0 \ +"$DIR/create_db.sh" + +MYSQL_CONTAINER=1 \ +DB_HOST="boulder-vitess" \ +DB_PORT=33577 \ +DB_CONFIG_FILE="${DIR}/../sa/db/dbconfig.mysql8.yml" \ +SKIP_CREATE=1 \ +SKIP_USERS=1 \ +"$DIR/create_db.sh" if [[ $# -eq 0 ]]; then exec python3 ./start.py diff --git a/test/integration/cert_storage_failed_test.go b/test/integration/cert_storage_failed_test.go index 603d3c342b3..5a6e7e55782 100644 --- a/test/integration/cert_storage_failed_test.go +++ b/test/integration/cert_storage_failed_test.go @@ -71,26 +71,34 @@ func getPrecertByName(db *sql.DB, reversedName string) (*x509.Certificate, error // assume exists (note that this different from the root program assumption // that a final certificate exists for any precertificate, though it is // similar in spirit). +// +// Note: For this test to support Vitess, it depends on a trigger being +// installed via test/vtcomboserver/install_trigger.sh, since Vitess does not +// support creating triggers via normal SQL commands. func TestIssuanceCertStorageFailed(t *testing.T) { os.Setenv("DIRECTORY", "http://boulder.service.consul:4001/directory") - ctx := context.Background() - db, err := sql.Open("mysql", vars.DBConnSAIntegrationFullPerms) test.AssertNotError(t, err, "failed to open db connection") - _, err = db.ExecContext(ctx, `DROP TRIGGER IF EXISTS fail_ready`) - test.AssertNotError(t, err, "failed to drop trigger") - - // Make a specific insert into certificates fail, for this test but not others. - // To limit the effect to this one test, we make the trigger aware of a specific - // hostname used in this test. Since the INSERT to the certificates table - // doesn't include the hostname, we look it up in the issuedNames table, keyed - // off of the serial. - // NOTE: CREATE and DROP TRIGGER do not work in prepared statements. Go's - // database/sql will automatically try to use a prepared statement if you pass - // any arguments to Exec besides the query itself, so don't do that. - _, err = db.ExecContext(ctx, ` + if os.Getenv("USE_VITESS") == "false" { + // This block is only necessary for ProxySQL + MariaDB and can be + // deleted once we're fully migrated to Vitess + MySQL 8, where the + // trigger is installed via test/vtcomboserver/install_trigger.sh. + + ctx := context.Background() + _, err = db.ExecContext(ctx, `DROP TRIGGER IF EXISTS fail_ready`) + test.AssertNotError(t, err, "failed to drop trigger") + + // Make a specific insert into certificates fail, for this test but not others. + // To limit the effect to this one test, we make the trigger aware of a specific + // hostname used in this test. Since the INSERT to the certificates table + // doesn't include the hostname, we look it up in the issuedNames table, keyed + // off of the serial. + // NOTE: CREATE and DROP TRIGGER do not work in prepared statements. Go's + // database/sql will automatically try to use a prepared statement if you pass + // any arguments to Exec besides the query itself, so don't do that. + _, err = db.ExecContext(ctx, ` CREATE TRIGGER fail_ready BEFORE INSERT ON certificates FOR EACH ROW BEGIN @@ -105,9 +113,10 @@ func TestIssuanceCertStorageFailed(t *testing.T) { END IF; END `) - test.AssertNotError(t, err, "failed to create trigger") + test.AssertNotError(t, err, "failed to create trigger") - defer db.ExecContext(ctx, `DROP TRIGGER IF EXISTS fail_ready`) + defer db.ExecContext(ctx, `DROP TRIGGER IF EXISTS fail_ready`) + } certKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) test.AssertNotError(t, err, "creating random cert key") diff --git a/test/proxysql/proxysql.cnf b/test/proxysql/proxysql.cnf index f918aa4538d..e8c7a68272a 100644 --- a/test/proxysql/proxysql.cnf +++ b/test/proxysql/proxysql.cnf @@ -56,7 +56,7 @@ mysql_variables = mysql_servers = ( { - address = "boulder-mysql"; + address = "boulder-mariadb"; port = 3306; hostgroup = 0; max_connections = 100; diff --git a/test/secrets/badkeyrevoker_dburl b/test/secrets/dburls/proxysql/badkeyrevoker_dburl similarity index 100% rename from test/secrets/badkeyrevoker_dburl rename to test/secrets/dburls/proxysql/badkeyrevoker_dburl diff --git a/test/secrets/cert_checker_dburl b/test/secrets/dburls/proxysql/cert_checker_dburl similarity index 100% rename from test/secrets/cert_checker_dburl rename to test/secrets/dburls/proxysql/cert_checker_dburl diff --git a/test/secrets/incidents_dburl b/test/secrets/dburls/proxysql/incidents_dburl similarity index 100% rename from test/secrets/incidents_dburl rename to test/secrets/dburls/proxysql/incidents_dburl diff --git a/test/secrets/revoker_dburl b/test/secrets/dburls/proxysql/revoker_dburl similarity index 100% rename from test/secrets/revoker_dburl rename to test/secrets/dburls/proxysql/revoker_dburl diff --git a/test/secrets/sa_dburl b/test/secrets/dburls/proxysql/sa_dburl similarity index 100% rename from test/secrets/sa_dburl rename to test/secrets/dburls/proxysql/sa_dburl diff --git a/test/secrets/sa_ro_dburl b/test/secrets/dburls/proxysql/sa_ro_dburl similarity index 100% rename from test/secrets/sa_ro_dburl rename to test/secrets/dburls/proxysql/sa_ro_dburl diff --git a/test/secrets/dburls/vitess/badkeyrevoker_dburl b/test/secrets/dburls/vitess/badkeyrevoker_dburl new file mode 100644 index 00000000000..55f17e4dc81 --- /dev/null +++ b/test/secrets/dburls/vitess/badkeyrevoker_dburl @@ -0,0 +1 @@ +badkeyrevoker@tcp(boulder-vitess:33577)/boulder_sa_integration diff --git a/test/secrets/dburls/vitess/cert_checker_dburl b/test/secrets/dburls/vitess/cert_checker_dburl new file mode 100644 index 00000000000..c229097c7ef --- /dev/null +++ b/test/secrets/dburls/vitess/cert_checker_dburl @@ -0,0 +1 @@ +cert_checker@tcp(boulder-vitess:33577)/boulder_sa_integration diff --git a/test/secrets/dburls/vitess/incidents_dburl b/test/secrets/dburls/vitess/incidents_dburl new file mode 100644 index 00000000000..94743bb6f0e --- /dev/null +++ b/test/secrets/dburls/vitess/incidents_dburl @@ -0,0 +1 @@ +incidents_sa@tcp(boulder-vitess:33577)/incidents_sa_integration?readTimeout=14s&timeout=1s diff --git a/test/secrets/dburls/vitess/revoker_dburl b/test/secrets/dburls/vitess/revoker_dburl new file mode 100644 index 00000000000..fb0d349b85d --- /dev/null +++ b/test/secrets/dburls/vitess/revoker_dburl @@ -0,0 +1 @@ +revoker@tcp(boulder-vitess:33577)/boulder_sa_integration diff --git a/test/secrets/dburls/vitess/sa_dburl b/test/secrets/dburls/vitess/sa_dburl new file mode 100644 index 00000000000..6b1837cb407 --- /dev/null +++ b/test/secrets/dburls/vitess/sa_dburl @@ -0,0 +1 @@ +sa@tcp(boulder-vitess:33577)/boulder_sa_integration?readTimeout=14s&writeTimeout=14s&timeout=1s diff --git a/test/secrets/dburls/vitess/sa_ro_dburl b/test/secrets/dburls/vitess/sa_ro_dburl new file mode 100644 index 00000000000..33056433cd7 --- /dev/null +++ b/test/secrets/dburls/vitess/sa_ro_dburl @@ -0,0 +1 @@ +sa_ro@tcp(boulder-vitess:33577)/boulder_sa_integration?readTimeout=14s&writeTimeout=14s&timeout=1s diff --git a/test/vtcomboserver/Dockerfile b/test/vtcomboserver/Dockerfile new file mode 100644 index 00000000000..a1e81c54f14 --- /dev/null +++ b/test/vtcomboserver/Dockerfile @@ -0,0 +1,65 @@ +# syntax=docker/dockerfile:1 +ARG VITESS_TAG=v22.0.0 +ARG MYSQL_BASE_IMAGE=mysql/mysql-server:8.0 + +FROM golang:1.25.2-bookworm AS build +ARG VITESS_TAG +ENV CGO_ENABLED=0 + +# Largely cobbled together from Vitess's own Dockerfiles and install scripts: +# - https://github.com/vitessio/vitess/blob/v22.0.1/docker/lite/Dockerfile +# - https://github.com/vitessio/vitess/blob/v22.0.1/docker/lite/Dockerfile.mysql84 +# - https://github.com/vitessio/vitess/blob/v22.0.1/docker/utils/install_dependencies.sh +# - https://github.com/vitessio/vitess/blob/v22.0.1/docker/vttestserver/Dockerfile.mysql80 + +ENV VTROOT=/vt +WORKDIR /src + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --depth 1 --branch "${VITESS_TAG}" https://github.com/vitessio/vitess.git . + +# Build only the binaries we need +RUN go build -trimpath -o /out/vtcombo ./go/cmd/vtcombo +RUN go build -trimpath -o /out/mysqlctl ./go/cmd/mysqlctl + +FROM ${MYSQL_BASE_IMAGE} AS runtime + +# Vitess expects to run as a non-root user +RUN groupadd -r vitess && useradd -r -g vitess vitess + +ENV VTROOT=/vt \ + VTDATAROOT=/vt/vtdataroot \ + PATH=/vt/bin:$PATH + +WORKDIR /vt + +RUN mkdir -p /vt/vtdataroot /vt/bin /vt/config/mycnf /vt/web \ + && chown -R vitess:vitess /vt + +# Copy only the binaries we need +COPY --from=build /out/vtcombo /vt/bin/vtcombo +COPY --from=build /out/mysqlctl /vt/bin/mysqlctl +RUN chmod +x /vt/bin/* && chown -R vitess:vitess /vt/bin + +# Copy other necessary files +COPY --from=build --chown=vitess:vitess /src/config/init_db.sql /vt/config/ +COPY --from=build --chown=vitess:vitess /src/config/mycnf /vt/config/mycnf +COPY --from=build --chown=vitess:vitess /src/web/vtadmin /vt/web/vtadmin + +# Copy install script we use to trigger failures in our integration tests +COPY --chown=vitess:vitess install_trigger.sh /vt/install_trigger.sh +RUN chmod +x /vt/install_trigger.sh + +VOLUME /vt/vtdataroot + +COPY setup_vschema_folder.sh /vt/setup_vschema_folder.sh +COPY run.sh /vt/run.sh +RUN chmod +x /vt/setup_vschema_folder.sh /vt/run.sh \ + && chown vitess:vitess /vt/setup_vschema_folder.sh /vt/run.sh + +USER vitess + +CMD ["/vt/run.sh", "8.0.40-Vitess"] diff --git a/test/vtcomboserver/install_trigger.sh b/test/vtcomboserver/install_trigger.sh new file mode 100644 index 00000000000..dbc2904284b --- /dev/null +++ b/test/vtcomboserver/install_trigger.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# +# Note: This script exists to support the integration test at +# test/integration/cert_storage_failed_test.go. Because Vitess doesn’t support +# creating triggers through normal SQL, this script waits for the Vitess +# database to come up and then installs a trigger that simulates an error when +# inserting into the certificates table under a specific condition. +# + +set -eu + +VT_DB="vt_${MYSQL_DATABASE:-boulder_sa_integration}_0" +SOCK="/vt/vtdataroot/vt_0000000001/mysql.sock" +MYSQL_ARGS=(-uroot -S "$SOCK") +TIMEOUT=120 + +# +# Helpers +# + +exit_msg() { + echo "$*" >&2 + exit 2 +} + +table_exists() { + local result + result="$(mysql "${MYSQL_ARGS[@]}" -Nse \ + "SELECT 1 FROM information_schema.tables WHERE table_schema='${VT_DB}' AND table_name='certificates' LIMIT 1" \ + || true)" + [ "$result" = "1" ] +} + +# Wait for the certificates table to be created +printf '[install_trigger] waiting for %s.certificates to appear...\n' "$VT_DB" + +i=0 +while [ "$i" -lt "$TIMEOUT" ] +do + if table_exists + then + break + fi + sleep 1 + i=$((i+1)) +done + +if ! table_exists +then + exit_msg "[install_trigger] ERROR: ${VT_DB}.certificates not found after ${TIMEOUT}s" +fi + +# Install trigger that simulates an error when inserting into the certificates +# table for TestIssuanceCertStorageFailed in /test/integration/cert_storage_failed_test.go. +printf '[install_trigger] installing trigger on %s.certificates\n' "$VT_DB" +mysql "${MYSQL_ARGS[@]}" "$VT_DB" <<'SQL' +DELIMITER $$ +DROP TRIGGER IF EXISTS fail_ready $$ +CREATE TRIGGER fail_ready +BEFORE INSERT ON certificates +FOR EACH ROW +BEGIN + DECLARE reversedName1 VARCHAR(255); + SELECT reversedName INTO reversedName1 + FROM issuedNames + WHERE serial = NEW.serial + AND reversedName LIKE 'com.wantserror.%'; + IF reversedName1 IS NOT NULL AND reversedName1 != '' THEN + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = 'Pretend there was an error inserting into certificates'; + END IF; +END $$ +DELIMITER ; +SQL + +printf '[install_trigger] done\n' diff --git a/test/vtcomboserver/run.sh b/test/vtcomboserver/run.sh new file mode 100755 index 00000000000..4fd7f96ecd5 --- /dev/null +++ b/test/vtcomboserver/run.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Much of the below is adapted from upstream Vitess's vttestserver run.sh +# but instead of using vttestserver, we use vtcombo directly: +# https://github.com/vitessio/vitess/blob/v22.0.1/docker/vttestserver/run.sh + +# Copyright 2021 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Setup the Vschema Folder +/vt/setup_vschema_folder.sh "$KEYSPACES" "$NUM_SHARDS" + +# Set the maximum connections in the cnf file +# use 1000 as the default if it is unspecified +if [[ -z $MYSQL_MAX_CONNECTIONS ]]; then + MYSQL_MAX_CONNECTIONS=1000 +fi +echo "max_connections = $MYSQL_MAX_CONNECTIONS" >> /vt/config/mycnf/test-suite.cnf + +# Delete socket files before running mysqlctld if exists. +# This is the primary reason for unhealthy state on restart. +# https://github.com/vitessio/vitess/pull/5115/files +rm -vf "$VTDATAROOT"/"$tablet_dir"/{mysql.sock,mysql.sock.lock} + +# Kick off script to install trigger we use to simulate +# errors in our integration tests, in the background. +/vt/install_trigger.sh & + +# Create Vitess JSON topo, start vtcombo and mysql. For more details see: +# - https://vitess.io/docs/22.0/reference/programs/vtcombo +# - https://github.com/vitessio/vitess/blob/v22.0.1/go/vt/vttest/vtprocess.go +# - https://github.com/vitessio/vitess/blob/v22.0.1/proto/vttest.proto +/vt/bin/vtcombo \ + --port "${PORT:-33574}" \ + --bind-address "${VTCOMBO_BIND_HOST:-0.0.0.0}" \ + --mysql_server_bind_address "${MYSQL_SERVER_BIND_ADDRESS:-0.0.0.0}" \ + --mysql_server_port "${MYSQL_SERVER_PORT:-33577}" \ + --mysql_auth_server_impl "none" \ + --mysql_server_version "${MYSQL_SERVER_VERSION:-8.0.40-Vitess}" \ + --db_charset "${CHARSET:-utf8mb4}" \ + --foreign_key_mode "${FOREIGN_KEY_MODE:-allow}" \ + --enable_online_ddl \ + --enable_direct_ddl \ + --planner-version "${PLANNER_VERSION:-gen4}" \ + --vschema_ddl_authorized_users "${VSCHEMA_DDL_AUTH_USERS:-%}" \ + --tablet_refresh_interval "${TABLET_REFRESH_INTERVAL:-10s}" \ + --schema_dir "/vt/schema" \ + --queryserver-config-max-result-size "${QUERY_MAX_RESULT_SIZE:-1000000}" \ + --queryserver-config-warn-result-size "${QUERY_WARN_RESULT_SIZE:-1000000}" \ + --normalize_queries \ + --queryserver-config-pool-size 64 \ + --queryserver-config-stream-pool-size 200 \ + --queryserver-config-transaction-cap 80 \ + --queryserver-config-query-timeout 300s \ + --queryserver-config-schema-reload-time 60s \ + --queryserver-config-txpool-timeout 300s \ + --json_topo "$(printf '{"cells":["test"],"keyspaces":[%s]}' \ + "$(IFS=, read -ra ks <<< "${KEYSPACES}"; \ + for i in "${!ks[@]}"; do \ + printf '%s{"name":"%s","shards":[{"name":"0"}]}' \ + "$([ $i -gt 0 ] && echo ,)" "${ks[$i]}"; \ + done)")" \ + --start_mysql diff --git a/test/vtcomboserver/setup_vschema_folder.sh b/test/vtcomboserver/setup_vschema_folder.sh new file mode 100755 index 00000000000..248e292b021 --- /dev/null +++ b/test/vtcomboserver/setup_vschema_folder.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copied from Vitess's own setup_vschema_folder.sh: +# https://github.com/vitessio/vitess/blob/v22.0.1/docker/vttestserver/setup_vschema_folder.sh + +# Copyright 2021 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# getLength gets the number of elements in a comma separated string +function getLength() { + COUNT=0 + for _ in ${1//,/ } + do + ((COUNT++)) + done + echo "$COUNT" +} + +# The first argument to the file is a comma separated list of keyspaces and the second argument is the comma separated list of number of shards. + +KEYSPACES="$1" +NUM_SHARDS="$2" + +COUNT_KEYSPACES=$(getLength "$KEYSPACES") +COUNT_NUM_SHARDS=$(getLength "$NUM_SHARDS") + +# Incase the number of keyspaces and num_shards do not match, throw an error +if [ "$COUNT_KEYSPACES" != "$COUNT_NUM_SHARDS" ]; then + echo "Incompatible list of keyspaces and number of shards" + exit 1 +fi + +# Convert the strings to lists +read -ra KEYSPACES_LIST <<<"${KEYSPACES//,/ }" +read -ra NUM_SHARDS_LIST <<<"${NUM_SHARDS//,/ }" + +# create the main schema directory +mkdir -p /vt/schema/ + +i=0; +for keyspace in "${KEYSPACES_LIST[@]}"; do + # create a directory for each keyspace + mkdir -p "/vt/schema/$keyspace" + num_shard=${NUM_SHARDS_LIST[$i]} + # Create a vschema.json file only if the number of shards are more than 1 + if [[ $num_shard -gt "1" ]]; then + printf "{\n\t\"sharded\": true\n}\n" > "/vt/schema/$keyspace/vschema.json" + fi + ((i++)) +done diff --git a/test/vtcomboserver/tag_and_upload.sh b/test/vtcomboserver/tag_and_upload.sh new file mode 100755 index 00000000000..56b4ffe1e4f --- /dev/null +++ b/test/vtcomboserver/tag_and_upload.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -feuxo pipefail + +cd $(dirname $0) + +DATESTAMP=$(date +%Y-%m-%d) +DOCKER_REPO="letsencrypt/boulder-vtcomboserver" +VITESS_TAG=v22.0.0 + +echo "Please login to allow push to DockerHub" +docker login + +# Usage: build_and_push_image $VITESS_TAG +build_and_push_image() { + VITESS_TAG="$1" + TAG_NAME="${DOCKER_REPO}:vitess${VITESS_TAG}_${DATESTAMP}" + echo "Building boulder-vtcomboserver image ${TAG_NAME}" + + # build, tag, and push the image. + docker buildx build \ + --build-arg "VITESS_TAG=${VITESS_TAG}" \ + --progress plain \ + --push \ + --tag "${TAG_NAME}" \ + --platform "linux/amd64" \ + . +} + +build_and_push_image $VITESS_TAG