Satisfy testing on macOS

.github/workflows/nightly.yml

@@ -0,0 +1,38 @@
+name: Nightly
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+
+jobs:
+  nightly:
+    name: "Python: ${{ matrix.python-version }}
+     SQLA: ${{ matrix.sqla-version }}
+     CrateDB: ${{ matrix.cratedb-version }}
+     on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        cratedb-version: ['nightly']
+        sqla-version: ['1.1.18', '1.2.19', '1.3.23']
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }}
+
+      - name: Invoke tests
+        run: |
+          bin/flake8
+          bin/coverage run bin/test -vv1

.github/workflows/tests.yml

@@ -0,0 +1,45 @@
+name: Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+
+jobs:
+  test:
+    name: "Python: ${{ matrix.python-version }}
+     SQLA: ${{ matrix.sqla-version }}
+     on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        cratedb-version: ['4.5.0']
+        sqla-version: ['1.1.18', '1.2.19', '1.3.23']
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Adjust environment for macOS
+        if: matrix.os == 'macos-latest'
+        run: |
+          brew install gnu-getopt
+          echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: |
+          ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }}
+
+      - name: Invoke tests
+        run: |
+          bin/flake8
+          bin/coverage run bin/test -vv1

base.cfg

@@ -19,11 +19,21 @@ eggs = crate
 recipe = zc.recipe.egg
 eggs = createcoverage
 
-[crate]
+[crate:linux]
 recipe = hexagonit.recipe.download
 url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz
 strip-top-level-dir = true
 
+[crate:macosx]
+recipe = hexagonit.recipe.download
+url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz
+strip-top-level-dir = true
+
+[crate:windows]
+recipe = hexagonit.recipe.download
+url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip
+strip-top-level-dir = true
+
 [test]
 relative-paths = true
 recipe = zc.recipe.testrunner

devtools/setup_ci.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -e
+
+function args() {
+  options=$(getopt --long cratedb-version: --long sqlalchemy-version: -- "$@")
+  [ $? -eq 0 ] || {
+    echo "Incorrect options provided"
+    exit 1
+  }
+  eval set -- "$options"
+  while true; do
+      case "$1" in
+      --cratedb-version)
+          shift;
+          cratedb_version=$1
+          ;;
+      --sqlalchemy-version)
+          shift;
+          sqlalchemy_version=$1
+          ;;
+      --)
+          shift
+          break
+          ;;
+      esac
+      shift
+  done
+}
+
+function main() {
+
+  # Read command line arguments.
+  args $0 "$@"
+
+  # Sanity checks.
+  [ -z ${cratedb_version} ] && {
+    echo "--cratedb-version must be given"
+    exit 1
+  }
+  [ -z ${sqlalchemy_version} ] && {
+    echo "--sqlalchemy-version must be given"
+    exit 1
+  }
+
+  # Let's go.
+  echo "Invoking tests with CrateDB ${cratedb_version} and SQLAlchemy ${sqlalchemy_version}"
+
+  python -m pip install --upgrade pip
+
+  # Workaround needed for Python 3.5
+  python -m pip install --upgrade "setuptools>=31,<51"
+
+  pip install zc.buildout==2.13.4
+
+  # Replace SQLAlchemy version.
+  sed -ir "s/SQLAlchemy.*/SQLAlchemy = ${sqlalchemy_version}/g" versions.cfg
+
+  # Replace CrateDB version.
+  if [ ${cratedb_version} = "nightly" ]; then
+    sed -ir "s/releases/releases\/nightly/g" base.cfg
+    sed -ir "s/crate_server.*/crate_server = latest/g" versions.cfg
+  else
+    sed -ir "s/crate_server.*/crate_server = ${cratedb_version}/g" versions.cfg
+  fi
+
+  buildout -n -c base.cfg
+
+}
+
+main "$@"

src/crate/client/sqlalchemy/doctests/itests.txt

@@ -49,10 +49,9 @@ Insert a new location::
     >>> session.add(location)
     >>> session.flush()
 
-Currently refresh option is missing, therefore sleep for now::
+Refresh "locations" table:
 
-    >>> from time import sleep
-    >>> sleep(1)
+    >>> _ = connection.execute("REFRESH TABLE locations")
 
 Inserted location is available::
 
@@ -106,7 +105,10 @@ The datetime and date can be set using a update statement::
     >>> location.nullable_date = datetime.today()
     >>> location.nullable_datetime = datetime.utcnow()
     >>> session.flush()
-    >>> sleep(1.1)  # wait for index refresh
+
+Refresh "locations" table:
+
+    >>> _ = connection.execute("REFRESH TABLE locations")
 
 Boolean values get set natively::
 
@@ -140,7 +142,11 @@ Update multiple Locations::
     ...     session.add(loc)
     ...     session.flush()
 
-    >>> sleep(2)  # give crate some time to settle
+Refresh "locations" table:
+
+    >>> _ = connection.execute("REFRESH TABLE locations")
+
+Query database:
 
     >>> result = connection.execute("update locations set flag=true where kind='Update'")
     >>> result.rowcount
@@ -154,7 +160,10 @@ documents in the table::
     True
 
     >>> session.commit()
-    >>> sleep(2)  # give crate some time to settle
+
+Refresh "locations" table:
+
+    >>> _ = connection.execute("REFRESH TABLE locations")
 
 Test that objects can be used as list too::
 
@@ -202,7 +211,10 @@ test updated nested dict::
     >>> char.details['name']['first'] = 'Trillian'
     >>> char.details['size'] = 45
     >>> session.commit()
-    >>> sleep(1.1)  # wait for index refresh
+
+Refresh "characters" table:
+
+    >>> _ = connection.execute("REFRESH TABLE characters")
 
     >>> session.refresh(char)
     >>> import pprint

src/crate/testing/doctests/layer.txt

@@ -225,26 +225,9 @@ We might have to wait a moment before the cluster is finally created::
 From Uri
 --------
 
-The CrateLayer can also be created by providing a URI that points to a Crate
+The CrateLayer can also be created by providing a URI that points to a CrateDB
 tarball::
 
-    >>> import urllib.request
-    >>> import json
-    >>> with urllib.request.urlopen('http://crate.io/versions.json') as response:
-    ...     versions = json.loads(response.read().decode())
-    ...     version = versions['crate_testing']
-
-    >>> uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version)
-    >>> tmpdir = tempfile.mkdtemp()
-    >>> layer = CrateLayer.from_uri(
-    ...             uri, name='crate-uri', http_port=42203, directory=tmpdir)
-    >>> layer.setUp()
-
-    >>> work_dir = os.path.join(tmpdir, 'crate-' + version)
-    >>> os.path.exists(work_dir)
-    True
-
-    >>> layer.tearDown()
-
-    >>> os.path.exists(work_dir)
-    False
+    uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version)
+    layer = CrateLayer.from_uri(
+                uri, name='crate-uri', http_port=42203, directory=tmpdir)

src/crate/testing/layer.py

@@ -72,6 +72,8 @@ def prepend_http(host):
 
 
 def _download_and_extract(uri, directory):
+    sys.stderr.write("\nINFO:    Downloading CrateDB archive from {} into {}".format(uri, directory))
+    sys.stderr.flush()
     with io.BytesIO(urlopen(uri).read()) as tmpfile:
         with tarfile.open(fileobj=tmpfile) as t:
             t.extractall(directory)
@@ -160,7 +162,8 @@ def from_uri(uri,
         crate_home = os.path.join(directory, crate_dir)
 
         if os.path.exists(crate_home):
-            sys.stderr.write('Not extracting Crate tarball because folder already exists')
+            sys.stderr.write("\nWARNING: Not extracting Crate tarball because folder already exists")
+            sys.stderr.flush()
         else:
             _download_and_extract(uri, directory)
 

src/crate/testing/test_layer.py

@@ -18,11 +18,15 @@
 # However, if you have executed another commercial license agreement
 # with Crate these terms will supersede the license and you may use the
 # software solely pursuant to the terms of the relevant commercial agreement.
-
+import json
 import os
 import tempfile
+import urllib
+from distutils.version import LooseVersion
 from unittest import TestCase, mock
 from io import BytesIO
+
+import crate
 from .layer import CrateLayer, prepend_http, http_url_from_host_port, wait_for_http_url
 
 
@@ -58,6 +62,22 @@ def test_wait_for_http(self):
         addr = wait_for_http_url(log=log, timeout=1)
         self.assertEqual(None, addr)
 
+    @mock.patch.object(crate.testing.layer, "_download_and_extract", lambda uri, directory: None)
+    def test_layer_from_uri(self):
+        """
+        The CrateLayer can also be created by providing an URI that points to
+        a CrateDB tarball.
+        """
+        with urllib.request.urlopen("https://crate.io/versions.json") as response:
+            versions = json.loads(response.read().decode())
+            version = versions["crate_testing"]
+
+        self.assertGreaterEqual(LooseVersion(version), LooseVersion("4.5.0"))
+
+        uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format(version)
+        layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203)
+        self.assertIsInstance(layer, CrateLayer)
+
     @mock.patch.dict('os.environ', {}, clear=True)
     def test_java_home_env_not_set(self):
         with tempfile.TemporaryDirectory() as tmpdir: