From f40db5dd55457a4631ae3a4ec522372a70a946db Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 28 Oct 2020 18:17:03 +0100 Subject: [PATCH 1/3] Satisfy testing on macOS --- .github/workflows/main.yml | 61 ------------------------ .github/workflows/nightly.yml | 38 +++++++++++++++ .github/workflows/tests.yml | 46 ++++++++++++++++++ base.cfg | 12 ++++- devtools/setup_ci.sh | 71 ++++++++++++++++++++++++++++ src/crate/testing/doctests/layer.txt | 25 ++-------- src/crate/testing/layer.py | 5 +- src/crate/testing/test_layer.py | 22 ++++++++- versions.cfg | 2 +- 9 files changed, 196 insertions(+), 86 deletions(-) delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/tests.yml create mode 100755 devtools/setup_ci.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index fd169617..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - workflow_dispatch: - schedule: - - cron: '0 2 * * *' - - -jobs: - test: - name: "Python: ${{ matrix.python-version }} - SQLA: ${{ matrix.sqla-version }} - CrateDB: ${{ matrix.crate-version }} - on ${{ matrix.os }}" - runs-on: ${{ matrix.os }} - strategy: - matrix: - crate-version: [nightly] - os: [ubuntu-latest] - sqla-version: ['1.1.18', '1.2.19', '1.3.23'] - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] - fail-fast: false - - steps: - - uses: actions/checkout@master - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - # Workaround needed for Python 3.5 - python -m pip install --upgrade "setuptools>=31,<51" - - pip install zc.buildout==2.13.4 - - # replace SQLAlchemy version - sed -ir 's/SQLAlchemy.*/SQLAlchemy = ${{ matrix.sqla-version }}/g' versions.cfg - - # replace CrateDB version - if [ ${{ matrix.crate-version }} = "nightly" ]; then - sed -ir 's/releases/releases\/nightly/g' base.cfg - sed -ir 's/crate_server.*/crate_server = latest/g' versions.cfg - else - sed -ir 's/crate-/crate_/g' base.cfg - sed -ir 's/crate_server.*/crate_server = ${{ matrix.crate-version }}/g' versions.cfg - fi - - buildout -n -c base.cfg - - - name: Test - run: | - bin/flake8 - bin/coverage run bin/test -vv1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..e81cfaa1 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,38 @@ +name: Nightly + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' + + +jobs: + nightly: + name: "Python: ${{ matrix.python-version }} + SQLA: ${{ matrix.sqla-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + cratedb-version: ['nightly'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] + fail-fast: false + + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} + + - name: Invoke tests + run: | + bin/flake8 + bin/coverage run bin/test -vv1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..883185ae --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + + +jobs: + test: + name: "Python: ${{ matrix.python-version }} + SQLA: ${{ matrix.sqla-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + cratedb-version: ['4.5.0'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] + fail-fast: false + + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Adjust environment for macOS + if: matrix.os == 'macos-latest' + run: | + brew install gnu-getopt + echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH + + - name: Install dependencies + run: | + ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} + + - name: Invoke tests + run: | + bin/flake8 + bin/coverage run bin/test -vv1 diff --git a/base.cfg b/base.cfg index f0df080e..e470e47b 100644 --- a/base.cfg +++ b/base.cfg @@ -19,11 +19,21 @@ eggs = crate recipe = zc.recipe.egg eggs = createcoverage -[crate] +[crate:linux] recipe = hexagonit.recipe.download url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz strip-top-level-dir = true +[crate:macosx] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz +strip-top-level-dir = true + +[crate:windows] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip +strip-top-level-dir = true + [test] relative-paths = true recipe = zc.recipe.testrunner diff --git a/devtools/setup_ci.sh b/devtools/setup_ci.sh new file mode 100755 index 00000000..ec70f22d --- /dev/null +++ b/devtools/setup_ci.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +function args() { + options=$(getopt --long cratedb-version: --long sqlalchemy-version: -- "$@") + [ $? -eq 0 ] || { + echo "Incorrect options provided" + exit 1 + } + eval set -- "$options" + while true; do + case "$1" in + --cratedb-version) + shift; + cratedb_version=$1 + ;; + --sqlalchemy-version) + shift; + sqlalchemy_version=$1 + ;; + --) + shift + break + ;; + esac + shift + done +} + +function main() { + + # Read command line arguments. + args $0 "$@" + + # Sanity checks. + [ -z ${cratedb_version} ] && { + echo "--cratedb-version must be given" + exit 1 + } + [ -z ${sqlalchemy_version} ] && { + echo "--sqlalchemy-version must be given" + exit 1 + } + + # Let's go. + echo "Invoking tests with CrateDB ${cratedb_version} and SQLAlchemy ${sqlalchemy_version}" + + python -m pip install --upgrade pip + + # Workaround needed for Python 3.5 + python -m pip install --upgrade "setuptools>=31,<51" + + pip install zc.buildout==2.13.4 + + # Replace SQLAlchemy version. + sed -ir "s/SQLAlchemy.*/SQLAlchemy = ${sqlalchemy_version}/g" versions.cfg + + # Replace CrateDB version. + if [ ${cratedb_version} = "nightly" ]; then + sed -ir "s/releases/releases\/nightly/g" base.cfg + sed -ir "s/crate_server.*/crate_server = latest/g" versions.cfg + else + sed -ir "s/crate_server.*/crate_server = ${cratedb_version}/g" versions.cfg + fi + + buildout -n -c base.cfg + +} + +main "$@" diff --git a/src/crate/testing/doctests/layer.txt b/src/crate/testing/doctests/layer.txt index 6a17dffd..2e4e67b4 100644 --- a/src/crate/testing/doctests/layer.txt +++ b/src/crate/testing/doctests/layer.txt @@ -225,26 +225,9 @@ We might have to wait a moment before the cluster is finally created:: From Uri -------- -The CrateLayer can also be created by providing a URI that points to a Crate +The CrateLayer can also be created by providing a URI that points to a CrateDB tarball:: - >>> import urllib.request - >>> import json - >>> with urllib.request.urlopen('http://crate.io/versions.json') as response: - ... versions = json.loads(response.read().decode()) - ... version = versions['crate_testing'] - - >>> uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version) - >>> tmpdir = tempfile.mkdtemp() - >>> layer = CrateLayer.from_uri( - ... uri, name='crate-uri', http_port=42203, directory=tmpdir) - >>> layer.setUp() - - >>> work_dir = os.path.join(tmpdir, 'crate-' + version) - >>> os.path.exists(work_dir) - True - - >>> layer.tearDown() - - >>> os.path.exists(work_dir) - False + uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version) + layer = CrateLayer.from_uri( + uri, name='crate-uri', http_port=42203, directory=tmpdir) diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 47578b82..3bd3fc99 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -72,6 +72,8 @@ def prepend_http(host): def _download_and_extract(uri, directory): + sys.stderr.write("\nINFO: Downloading CrateDB archive from {} into {}".format(uri, directory)) + sys.stderr.flush() with io.BytesIO(urlopen(uri).read()) as tmpfile: with tarfile.open(fileobj=tmpfile) as t: t.extractall(directory) @@ -160,7 +162,8 @@ def from_uri(uri, crate_home = os.path.join(directory, crate_dir) if os.path.exists(crate_home): - sys.stderr.write('Not extracting Crate tarball because folder already exists') + sys.stderr.write("\nWARNING: Not extracting Crate tarball because folder already exists") + sys.stderr.flush() else: _download_and_extract(uri, directory) diff --git a/src/crate/testing/test_layer.py b/src/crate/testing/test_layer.py index f2bcfc4b..ca55a2f9 100644 --- a/src/crate/testing/test_layer.py +++ b/src/crate/testing/test_layer.py @@ -18,11 +18,15 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. - +import json import os import tempfile +import urllib +from distutils.version import LooseVersion from unittest import TestCase, mock from io import BytesIO + +import crate from .layer import CrateLayer, prepend_http, http_url_from_host_port, wait_for_http_url @@ -58,6 +62,22 @@ def test_wait_for_http(self): addr = wait_for_http_url(log=log, timeout=1) self.assertEqual(None, addr) + @mock.patch.object(crate.testing.layer, "_download_and_extract", lambda uri, directory: None) + def test_layer_from_uri(self): + """ + The CrateLayer can also be created by providing an URI that points to + a CrateDB tarball. + """ + with urllib.request.urlopen("https://crate.io/versions.json") as response: + versions = json.loads(response.read().decode()) + version = versions["crate_testing"] + + self.assertGreaterEqual(LooseVersion(version), LooseVersion("4.5.0")) + + uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format(version) + layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203) + self.assertIsInstance(layer, CrateLayer) + @mock.patch.dict('os.environ', {}, clear=True) def test_java_home_env_not_set(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/versions.cfg b/versions.cfg index cd8759f6..5f49dcc5 100644 --- a/versions.cfg +++ b/versions.cfg @@ -1,5 +1,5 @@ [versions] -crate_server = 4.4.2 +crate_server = 4.5.0 flake8 = 3.7.9 mccabe = 0.6.1 From b138f8276dece59b09c2c98716c4e8a086f204bb Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Fri, 26 Mar 2021 12:31:19 +0100 Subject: [PATCH 2/3] Tests: Improve stability by using "REFRESH TABLE" appropriately --- .../client/sqlalchemy/doctests/itests.txt | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/crate/client/sqlalchemy/doctests/itests.txt b/src/crate/client/sqlalchemy/doctests/itests.txt index 646173bb..1b07c085 100644 --- a/src/crate/client/sqlalchemy/doctests/itests.txt +++ b/src/crate/client/sqlalchemy/doctests/itests.txt @@ -49,10 +49,9 @@ Insert a new location:: >>> session.add(location) >>> session.flush() -Currently refresh option is missing, therefore sleep for now:: +Refresh "locations" table: - >>> from time import sleep - >>> sleep(1) + >>> _ = connection.execute("REFRESH TABLE locations") Inserted location is available:: @@ -106,7 +105,10 @@ The datetime and date can be set using a update statement:: >>> location.nullable_date = datetime.today() >>> location.nullable_datetime = datetime.utcnow() >>> session.flush() - >>> sleep(1.1) # wait for index refresh + +Refresh "locations" table: + + >>> _ = connection.execute("REFRESH TABLE locations") Boolean values get set natively:: @@ -140,7 +142,11 @@ Update multiple Locations:: ... session.add(loc) ... session.flush() - >>> sleep(2) # give crate some time to settle +Refresh "locations" table: + + >>> _ = connection.execute("REFRESH TABLE locations") + +Query database: >>> result = connection.execute("update locations set flag=true where kind='Update'") >>> result.rowcount @@ -154,7 +160,10 @@ documents in the table:: True >>> session.commit() - >>> sleep(2) # give crate some time to settle + +Refresh "locations" table: + + >>> _ = connection.execute("REFRESH TABLE locations") Test that objects can be used as list too:: @@ -202,7 +211,10 @@ test updated nested dict:: >>> char.details['name']['first'] = 'Trillian' >>> char.details['size'] = 45 >>> session.commit() - >>> sleep(1.1) # wait for index refresh + +Refresh "characters" table: + + >>> _ = connection.execute("REFRESH TABLE characters") >>> session.refresh(char) >>> import pprint From 2660acc330de2eed1d093576f028abd3e4983979 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Mon, 29 Mar 2021 14:43:27 +0200 Subject: [PATCH 3/3] Tests: Don't include CrateDB version into title of CI/GHA step Having it included, it would need maintenance on the branch protection page each time we bump the CrateDB version. --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 883185ae..3d7dabc6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,6 @@ jobs: test: name: "Python: ${{ matrix.python-version }} SQLA: ${{ matrix.sqla-version }} - CrateDB: ${{ matrix.cratedb-version }} on ${{ matrix.os }}" runs-on: ${{ matrix.os }} strategy: