Production-level support for MSSQL (apache#18382)

MSSQL has been somewhat experimental in the `main` branch, but as
we near releasing for 2.2.0 version, the image should support
the mssql at the level as it supports other databases.

This PR adds proper support for both PROD and CI images.

Dockerfile

@@ -134,6 +134,7 @@ RUN mkdir -pv /usr/share/man/man1 \
     && rm -rf /var/lib/apt/lists/*
 
 ARG INSTALL_MYSQL_CLIENT="true"
+ARG INSTALL_MSSQL_CLIENT="true"
 ARG AIRFLOW_REPO=apache/airflow
 ARG AIRFLOW_BRANCH=main
 ARG AIRFLOW_EXTRAS
@@ -174,6 +175,7 @@ ARG AIRFLOW_SOURCES_FROM="empty"
 ARG AIRFLOW_SOURCES_TO="/empty"
 
 ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
+    INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \
     AIRFLOW_REPO=${AIRFLOW_REPO} \
     AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \
     AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \
@@ -192,7 +194,9 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
     UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
 
 COPY scripts/docker/*.sh /scripts/docker/
-RUN bash ./scripts/docker/install_mysql.sh dev
+RUN bash ./scripts/docker/install_mysql.sh dev \
+    && bash ./scripts/docker/install_mssql.sh
+ENV PATH=${PATH}:/opt/mssql-tools/bin
 
 COPY docker-context-files /docker-context-files
 
@@ -374,6 +378,7 @@ ARG RUNTIME_APT_COMMAND="echo"
 ARG ADDITIONAL_RUNTIME_APT_COMMAND=""
 ARG ADDITIONAL_RUNTIME_APT_ENV=""
 ARG INSTALL_MYSQL_CLIENT="true"
+ARG INSTALL_MSSQL_CLIENT="true"
 ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 ARG AIRFLOW_HOME
 # Having the variable in final image allows to disable providers manager warnings when
@@ -391,6 +396,7 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
     RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} \
     ADDITIONAL_RUNTIME_APT_COMMAND=${ADDITIONAL_RUNTIME_APT_COMMAND} \
     INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
+    INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \
     AIRFLOW_UID=${AIRFLOW_UID} AIRFLOW_GID=${AIRFLOW_GID} \
     AIRFLOW__CORE__LOAD_EXAMPLES="false" \
     AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
@@ -418,12 +424,15 @@ RUN mkdir -pv /usr/share/man/man1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Only copy install_mysql and install_pip_version.sh. We do not need any other scripts in the final image.
-COPY scripts/docker/install_mysql.sh scripts/docker/install_pip_version.sh /scripts/docker/
+# Only copy install_m(y/s)sql and install_pip_version.sh. We do not need any other scripts in the final image.
+COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh scripts/docker/install_pip_version.sh \
+   /scripts/docker/
 
 # fix permission issue in Azure DevOps when running the scripts
 RUN chmod a+x /scripts/docker/install_mysql.sh && \
     /scripts/docker/install_mysql.sh prod && \
+    chmod a+x /scripts/docker/install_mssql.sh && \
+    /scripts/docker/install_mssql.sh && \
     addgroup --gid "${AIRFLOW_GID}" "airflow" && \
     adduser --quiet "airflow" --uid "${AIRFLOW_UID}" \
         --gid "${AIRFLOW_GID}" \

Dockerfile.ci

@@ -104,6 +104,7 @@ RUN mkdir -pv /usr/share/man/man1 \
 
 COPY scripts/docker/*.sh /scripts/docker/
 RUN bash /scripts/docker/install_mysql.sh dev \
+    && bash /scripts/docker/install_mssql.sh \
     && adduser airflow \
     && echo "airflow:airflow" | chpasswd \
     && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \
@@ -178,17 +179,6 @@ RUN mkdir -pv /usr/share/man/man1 \
     && apt-get autoremove -yqq --purge \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
-    && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
-    && curl https://packages.microsoft.com/config/debian/9/prod.list > /etc/apt/sources.list.d/mssql-release.list \
-    && apt-get update -yqq \
-    && apt-get upgrade -yqq \
-    && ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends \
-      gcc \
-      unixodbc-dev  \
-      g++ \
-      msodbcsql17 \
-      mssql-tools \
-    && rm -rf /var/lib/apt/lists/* \
     && curl https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz \
     |  tar -C /usr/bin --strip-components=1 -xvzf - docker/docker
 
@@ -262,12 +252,13 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
 # In the CI image we always:
-# * install MySQL
+# * install MySQL, MsSQL
 # * install airflow from current sources, not from PyPI package
 # * install airflow without `--user` flag
 # * install airflow in editable mode
 # * install always current version of airflow
     INSTALL_MYSQL_CLIENT="true" \
+    INSTALL_MSSQL_CLIENT="true" \
     AIRFLOW_INSTALLATION_METHOD="." \
     AIRFLOW_INSTALL_USER_FLAG="" \
     AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \

breeze

@@ -1094,6 +1094,11 @@ function breeze::parse_arguments() {
             echo "Install MySQL client: ${INSTALL_MYSQL_CLIENT}"
             shift
             ;;
+        --disable-mssql-client-installation)
+            export INSTALL_MSSQL_CLIENT="false"
+            echo "Install MsSQL client: ${INSTALL_MSSQL_CLIENT}"
+            shift
+            ;;
         --constraints-location)
             export AIRFLOW_CONSTRAINTS_LOCATION="${2}"
             echo "Constraints location: ${AIRFLOW_CONSTRAINTS_LOCATION}"

docs/apache-airflow/howto/set-up-database.rst

@@ -32,10 +32,10 @@ By default, Airflow uses **SQLite**, which is intended for development purposes
 
 Airflow supports the following database engine versions, so make sure which version you have. Old versions may not support all SQL statements.
 
-  * PostgreSQL:  9.6, 10, 11, 12, 13
-  * MySQL: 5.7, 8
-  * MsSQL: 2017, 2019
-  * SQLite: 3.15.0+
+* PostgreSQL:  9.6, 10, 11, 12, 13
+* MySQL: 5.7, 8
+* MsSQL: 2017, 2019
+* SQLite: 3.15.0+
 
 If you plan on running more than one scheduler, you have to meet additional requirements.
 For details, see :ref:`Scheduler HA Database Requirements <scheduler:ha:db_requirements>`.
@@ -261,6 +261,13 @@ You can read more about transaction isolation and snapshot features at
    GRANT ALL PRIVILEGES ON DATABASE airflow TO airflow_user;
 
 
+We recommend using the ``mssql+pyodbc`` driver and specifying it in your SqlAlchemy connection string.
+
+.. code-block:: text
+
+    mssql+pyodbc://<user>:<password>@<host>
+
+
 Other configuration options
 ---------------------------
 

docs/docker-stack/build-arg-ref.rst

@@ -124,6 +124,8 @@ for examples of using those arguments.
 |                                          |                                          | The mysql extra is removed from extras   |
 |                                          |                                          | if the client is not installed.          |
 +------------------------------------------+------------------------------------------+------------------------------------------+
+| ``INSTALL_MSSQL_CLIENT``                 | ``true``                                 | Whether MsSQL client should be installed |
++------------------------------------------+------------------------------------------+------------------------------------------+
 
 Installing Airflow using different methods
 ..........................................

docs/docker-stack/build.rst

@@ -607,7 +607,9 @@ where you can build the image using the packages downloaded by passing those bui
 * ``AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"``  - to not pre-cache packages from PyPI when building image
 * ``AIRFLOW_CONSTRAINTS_LOCATION=/docker-context-files/YOUR_CONSTRAINT_FILE.txt`` - to downloaded constraint files
 * (Optional) ``INSTALL_MYSQL_CLIENT="false"`` if you do not want to install ``MySQL``
-  client from the Oracle repositories. In this case also make sure that your
+  client from the Oracle repositories.
+* (Optional) ``INSTALL_MSSQL_CLIENT="false"`` if you do not want to install ``MsSQL``
+  client from the Microsoft repositories.
 
 Note, that the solution we have for installing python packages from local packages, only solves the problem
 of "air-gaped" python installation. The Docker image also downloads ``apt`` dependencies and ``node-modules``.

docs/docker-stack/docker-examples/restricted/restricted_environments.sh

@@ -41,6 +41,7 @@ docker build . \
     --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
     --build-arg AIRFLOW_VERSION="2.2.0.dev0" \
     --build-arg INSTALL_MYSQL_CLIENT="false" \
+    --build-arg INSTALL_MSSQL_CLIENT="false" \
     --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" \
     --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="true" \
     --build-arg AIRFLOW_CONSTRAINTS_LOCATION="/docker-context-files/constraints-3.7.txt" \

scripts/ci/libraries/_build_images.sh

@@ -810,6 +810,7 @@ function build_images::build_prod_images() {
         "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
         --build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \
         --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
+        --build-arg INSTALL_MSSQL_CLIENT="${INSTALL_MSSQL_CLIENT}" \
         --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
         --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \
         --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
@@ -845,6 +846,7 @@ function build_images::build_prod_images() {
         "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
         --build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \
         --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
+        --build-arg INSTALL_MSSQL_CLIENT="${INSTALL_MSSQL_CLIENT}" \
         --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
         --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
         --build-arg INSTALL_PROVIDERS_FROM_SOURCES="${INSTALL_PROVIDERS_FROM_SOURCES}" \

scripts/ci/libraries/_initialization.sh

@@ -395,6 +395,8 @@ function initialization::initialize_image_build_variables() {
     export AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES:="true"}"
     # by default install mysql client
     export INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT:="true"}
+    # by default install mssql client
+    export INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT:="true"}
     # additional tag for the image
     export IMAGE_TAG=${IMAGE_TAG:=""}
 

scripts/docker/common.sh

@@ -18,6 +18,7 @@
 set -euo pipefail
 
 test -v INSTALL_MYSQL_CLIENT
+test -v INSTALL_MSSQL_CLIENT
 test -v AIRFLOW_INSTALL_USER_FLAG
 test -v AIRFLOW_REPO
 test -v AIRFLOW_BRANCH

scripts/docker/install_mssql.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -exuo pipefail
+function install_mssql_client() {
+    echo
+    echo Installing mssql client
+    echo
+    curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+    curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
+    apt-get update -yqq
+    apt-get upgrade -yqq
+    ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends msodbcsql17 mssql-tools
+    rm -rf /var/lib/apt/lists/*
+    apt-get autoremove -yqq --purge
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+}
+
+# Install MsSQL client from Microsoft repositories
+if [[ ${INSTALL_MSSQL_CLIENT:="true"} == "true" ]]; then
+    install_mssql_client "${@}"
+fi

scripts/docker/install_mysql.sh

@@ -15,9 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-set -euo pipefail
-
-set -x
+set -exuo pipefail
 declare -a packages
 
 MYSQL_VERSION="8.0"
@@ -63,11 +61,6 @@ install_mysql_client() {
     apt-get clean && rm -rf /var/lib/apt/lists/*
 }
 
-
-
-# Install MySQL Client during the container build
-set -euo pipefail
-
 # Install MySQL client from Oracle repositories (Debian installs mariadb)
 # But only if it is not disabled
 if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then

scripts/in_container/prod/entrypoint_prod.sh

@@ -99,6 +99,8 @@ function wait_for_connection {
             detected_port=5432
         elif [[ ${BACKEND} == "mysql"* ]]; then
             detected_port=3306
+        elif [[ ${BACKEND} == "mssql"* ]]; then
+            detected_port=1433
         elif [[ ${BACKEND} == "redis"* ]]; then
             detected_port=6379
         elif [[ ${BACKEND} == "amqp"* ]]; then