From 460c69d2be5616561bb7f214636b453482409bd6 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Tue, 21 Oct 2025 15:41:17 -0400 Subject: [PATCH] Drop support for MySQL According to Chris Matthews, this had been added exclusively for the (now defunct) lnt.llvm.org instance, which was running on Google's internal infrastructure and hence had constraints on the database software it could use. Since that is not relevant anymore, simplify the codebase and the support matrix of LNT by dropping support for MySQL. According to Chris, LNT was originally always intended to run on a Postgres database. This allows us to actually test what we claim to support, making LNT more robust despite its slightly shrunken support matrix. In practice, I expect that nobody will be impacted by this change since nobody is running a MySQL instance. --- docs/developer_guide.rst | 8 +- docs/intro.rst | 9 +- lnt/server/db/migrations/upgrade_0_to_1.py | 3 - lnt/server/db/migrations/upgrade_13_to_14.py | 22 +---- lnt/server/db/testsuitedb.py | 5 -- tests/SharedInputs/mysql_wrapper.sh | 86 -------------------- tests/lit.cfg | 5 -- tests/lnttool/MySQLDB.shtest | 15 ---- tests/lnttool/import.shtest | 12 +++ 9 files changed, 19 insertions(+), 146 deletions(-) delete mode 100755 tests/SharedInputs/mysql_wrapper.sh delete mode 100644 tests/lnttool/MySQLDB.shtest create mode 100644 tests/lnttool/import.shtest diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst index 06879d99..1cdfac59 100644 --- a/docs/developer_guide.rst +++ b/docs/developer_guide.rst @@ -51,12 +51,6 @@ Optional Tests Some tests require additional tools to be installed and are not enabled by default. You can enable them by passing additional flags to ``lit``: - ``-Dmysql=1`` - Enable mysql database support testing. This requires MySQL-python to be - installed and expects the ``mysqld`` and ``mysqladmin`` binaries in your path. - Note that you do not need to setup an actual server, the tests will create - temporary instances on demand. - ``-Dtidylib=1`` Check generated html pages for errors using ``tidy-html5``. This requires ``pytidylib`` and ``tidy-html5`` to be installed. @@ -68,7 +62,7 @@ default. You can enable them by passing additional flags to ``lit``: Example:: - lit -sv -Dmysql=1 -Dtidylib=1 ./tests + lit -sv -Dtidylib=1 ./tests Publishing a new version of LNT ------------------------------- diff --git a/docs/intro.rst b/docs/intro.rst index 7a938305..97ae505d 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -86,17 +86,16 @@ for the templating engine. My hope is to eventually move to a more AJAXy web interface. The database layer uses SQLAlchemy for its ORM, and is typically backed by -SQLite, although I have tested on MySQL on the past, and supporting other -databases should be trivial. My plan is to always support SQLite as this allows -the possibility of developers easily running their own LNT installation for -viewing nightly test results, and to run with whatever DB makes the most sense +SQLite for local installations and PostgreSQL in production. The plan is to +always support SQLite as this allows developers to easily run their own LNT +installation locally, and to run with whatever DB makes the most sense on the server. Running a LNT Server Locally ---------------------------- LNT can accommodate many more users in the production config. In production: -- Postgres or MySQL should be used as the database. +- Postgres should be used as the database. - A proper wsgi server should be used, in front of a proxy like Nginx or Apache. To install the extra packages for the server config:: diff --git a/lnt/server/db/migrations/upgrade_0_to_1.py b/lnt/server/db/migrations/upgrade_0_to_1.py index b192c701..306bceff 100644 --- a/lnt/server/db/migrations/upgrade_0_to_1.py +++ b/lnt/server/db/migrations/upgrade_0_to_1.py @@ -183,8 +183,6 @@ def get_base_for_testsuite(test_suite): class Machine(UpdatedBase): # type: ignore[misc,valid-type] __tablename__ = db_key_name + '_Machine' - # For case sensitive compare. - __table_args__ = {'mysql_collate': 'utf8_bin'} id = Column("ID", Integer, primary_key=True) name = Column("Name", String(256), index=True) @@ -249,7 +247,6 @@ class Run(UpdatedBase): # type: ignore[misc,valid-type] class Test(UpdatedBase): # type: ignore[misc,valid-type] __tablename__ = db_key_name + '_Test' - __table_args__ = {'mysql_collate': 'utf8_bin'} id = Column("ID", Integer, primary_key=True) name = Column("Name", String(256), unique=True, index=True) diff --git a/lnt/server/db/migrations/upgrade_13_to_14.py b/lnt/server/db/migrations/upgrade_13_to_14.py index 4e09ce71..d32ba83b 100644 --- a/lnt/server/db/migrations/upgrade_13_to_14.py +++ b/lnt/server/db/migrations/upgrade_13_to_14.py @@ -6,7 +6,7 @@ # always the same so we have to rename from `Compile_XXXX` to `compile_XXX`. import collections -from sqlalchemy import delete, select, update, func, and_ +from sqlalchemy import delete, select, update, func from lnt.server.db.migrations.util import introspect_table, rename_table @@ -29,8 +29,7 @@ def _drop_suite(trans, name, engine): drop_fields(engine, test_suite_id, 'TestSuiteOrderFields', trans) drop_fields(engine, test_suite_id, 'TestSuiteMachineFields', trans) drop_fields(engine, test_suite_id, 'TestSuiteRunFields', trans) - - drop_samples_fields(engine, test_suite_id, trans) + drop_fields(engine, test_suite_id, 'TestSuiteSampleFields', trans) trans.execute(delete(test_suite).where(test_suite.c.Name == name)) @@ -45,23 +44,6 @@ def drop_fields(engine, test_suite_id, name, trans): return fields_table -def drop_samples_fields(engine, test_suite_id, trans): - """In the TestSuiteSampleFields, drop entries related to the test_suite_id. - - This extra function is needed because in MySQL it can't sort out the forign - keys in the same table. - """ - samples_table = introspect_table(engine, 'TestSuiteSampleFields') - order_files = delete(samples_table, - and_(samples_table.c.TestSuiteID == test_suite_id, - samples_table.c.status_field.isnot(None))) - trans.execute(order_files) - order_files = delete(samples_table, - samples_table.c.TestSuiteID == test_suite_id) - trans.execute(order_files) - return samples_table - - TableRename = collections.namedtuple('TableRename', 'old_name new_name') diff --git a/lnt/server/db/testsuitedb.py b/lnt/server/db/testsuitedb.py index 8b2a5a38..3dc64194 100644 --- a/lnt/server/db/testsuitedb.py +++ b/lnt/server/db/testsuitedb.py @@ -101,7 +101,6 @@ def set_fields_pop(self, data_dict): class Machine(self.base, ParameterizedMixin): __tablename__ = db_key_name + '_Machine' - __table_args__ = {'mysql_collate': 'utf8_bin'} DEFAULT_BASELINE_REVISION = v4db.baseline_revision fields = self.machine_fields @@ -427,8 +426,6 @@ def __json__(self, flatten_order=True): class Test(self.base, ParameterizedMixin): __tablename__ = db_key_name + '_Test' - # utf8_bin for case sensitive compare - __table_args__ = {'mysql_collate': 'utf8_bin'} id = Column("ID", Integer, primary_key=True) name = Column("Name", String(256), unique=True, index=True) @@ -748,7 +745,6 @@ def __repr__(self): class Baseline(self.base, ParameterizedMixin): """Baselines to compare runs to.""" __tablename__ = db_key_name + '_Baseline' - __table_args__ = {'mysql_collate': 'utf8_bin'} id = Column("ID", Integer, primary_key=True) name = Column("Name", String(32), unique=True) @@ -1033,7 +1029,6 @@ def _getOrCreateRun(self, session, run_data, machine, merge): def _importSampleValues(self, session, tests_data, run, config): # Load a map of all the tests, which we will extend when we find tests # that need to be added. - # Downcast to str, so we match on MySQL. test_cache = dict((test.name, test) for test in session.query(self.Test)) diff --git a/tests/SharedInputs/mysql_wrapper.sh b/tests/SharedInputs/mysql_wrapper.sh deleted file mode 100755 index 81ed3658..00000000 --- a/tests/SharedInputs/mysql_wrapper.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/sh -# This works like postgres_wrapper.sh but for mysql. -# -# This will bind to localhost on the specified port. There is a database -# called testdb; user 'root' can login without a password. -# -# Example: -# ./mysql_wrapper /tmp/myinstance 9101 mysql -u root -P 9101 testdb -# -# Inspired by https://github.com/tk0miya/testing.mysqld -set -eu - -DBDIR="$1" -shift -if [ -d "${DBDIR}" ]; then - echo 1>&2 "${DBDIR} already exists" - exit 1 -fi -PORT="$1" -shift - -mkdir -p "${DBDIR}" -mkdir -p "${DBDIR}/etc" -mkdir -p "${DBDIR}/data" -cat > "${DBDIR}/etc/my.cnf" << __EOF__ -[mysqld] -bind-address = 127.0.0.1 -port = ${PORT} -user = $(whoami) -__EOF__ - -MYSQL_INSTALL_DB_FLAGS+=" --defaults-file=\"${DBDIR}/etc/my.cnf\"" -MYSQL_INSTALL_DB_FLAGS+=" --initialize-insecure" -MYSQL_INSTALL_DB_FLAGS+=" --user=$(whoami)" -MYSQL_INSTALL_DB_FLAGS+=" --datadir=\"${DBDIR}/data\"" -MYSQL_INSTALL_DB_FLAGS+=" >& \"${DBDIR}/install_db.log\"" -echo "$ mysqld ${MYSQL_INSTALL_DB_FLAGS}" -eval mysqld ${MYSQL_INSTALL_DB_FLAGS} - - -MYSQLD_FLAGS+=" --defaults-file=\"${DBDIR}/etc/my.cnf\"" -MYSQLD_FLAGS+=" --datadir=\"${DBDIR}/data\"" -MYSQLD_FLAGS+=" --pid-file=\"${DBDIR}/mysqld.pid\"" -MYSQLD_FLAGS+=" --log-error=\"${DBDIR}/mysqld.error.log\"" -MYSQLD_FLAGS+="&" -echo "$ mysqld ${MYSQLD_FLAGS}" -eval mysqld ${MYSQLD_FLAGS} - -MYSQLADMIN_FLAGS+=" --defaults-file=\"${DBDIR}/etc/my.cnf\"" -MYSQLADMIN_FLAGS+=" -h 127.0.0.1" -MYSQLADMIN_FLAGS+=" -P ${PORT}" -MYSQLADMIN_FLAGS+=" -u root" - -# Poll server for 10 seconds to see when it is up. -set +e -for i in {1..100} -do - sleep 0.1 - echo "$ mysqladmin ${MYSQLADMIN_FLAGS} status" - eval mysqladmin ${MYSQLADMIN_FLAGS} status - if [ $? -eq 0 ]; then - break - fi -done -set -e - -# Drop database to be sure. The DB may not exist if the test has not been run -# before: hide output to avoid confusion. -set +e -echo "$ mysqladmin ${MYSQLADMIN_FLAGS} drop --force testdb" -eval mysqladmin ${MYSQLADMIN_FLAGS} drop --force testdb >& /dev/null -set -e - -echo "$ mysqladmin ${MYSQLADMIN_FLAGS} create testdb" -eval mysqladmin ${MYSQLADMIN_FLAGS} create testdb - -# Execute command -eval "$@" -RC=$? - -# Kill server -MYSQLD_PID="$(cat "${DBDIR}/mysqld.pid")" -kill -15 ${MYSQLD_PID} -[ $? -ne 0 ] && (echo 1>&1 "Could not kill mysql server"; exit 1) -wait "${MYSQLD_PID}" -exit $RC diff --git a/tests/lit.cfg b/tests/lit.cfg index cb78e865..44561795 100644 --- a/tests/lit.cfg +++ b/tests/lit.cfg @@ -53,11 +53,6 @@ postgres = shutil.which('postgres') if postgres is not None: config.available_features.add('postgres') -# Enable MySQL testing. This requires mysqld and mysqladmin binaries in PATH. -# (You do not need to start a server, the tests will create ad-hoc instances). -if lit_config.params.get('mysql', None): - config.available_features.add('mysql') - # Enable tidylib testing. This requires pytidylib and tidy-html5. if lit_config.params.get('tidylib', None): config.substitutions.append(('%{tidylib}', '--use-tidylib')) diff --git a/tests/lnttool/MySQLDB.shtest b/tests/lnttool/MySQLDB.shtest deleted file mode 100644 index 8ffc71e3..00000000 --- a/tests/lnttool/MySQLDB.shtest +++ /dev/null @@ -1,15 +0,0 @@ -# REQUIRES: mysql -# RUN: rm -rf "%t.install" -# RUN: %{shared_inputs}/mysql_wrapper.sh "%t.install" 9109 /bin/sh %s "%t.install" "%{shared_inputs}" -set -eux - -TESTDIR="$1" -SHARED_INPUTS="$2" - -lnt create "${TESTDIR}/instance" --db-dir mysql://root@localhost:9109 --default-db testdb --show-sql - -# Import a test set. -lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-a-small.plist" --show-sample-count -lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-b-small.plist" --show-sample-count -lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-arm.json" --format=json --show-sample-count -lnt import "${TESTDIR}/instance" "${SHARED_INPUTS}/sample-arm2.json" --format=json --show-sample-count diff --git a/tests/lnttool/import.shtest b/tests/lnttool/import.shtest new file mode 100644 index 00000000..29afa79e --- /dev/null +++ b/tests/lnttool/import.shtest @@ -0,0 +1,12 @@ +#!/bin/bash + +# RUN: rm -rf %t.instance +# RUN: lnt create %t.instance + +# Make sure we can import a basic test set +# RUN: lnt import %t.instance %{shared_inputs}/sample-a-small.plist + +# Make sure we can import test sets where the only difference is the +# case used in the test name. +# RUN: lnt import %t.instance %{shared_inputs}/sample-arm.json --format=json +# RUN: lnt import %t.instance %{shared_inputs}/sample-arm2.json --format=json