diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 621d6850..60e4e377 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,11 +16,11 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-2019] python-version: ['3.7', '3.8', '3.9', '3.10'] cratedb-version: ['4.8.0'] sqla-version: ['1.3.24'] - fail-fast: true + fail-fast: false steps: - uses: actions/checkout@v3 @@ -36,13 +36,37 @@ jobs: echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH - name: Install dependencies + shell: bash run: | ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} + - name: Fix Windows environment + if: matrix.os == 'windows-2019' + shell: bash + run: | + # Disable firewall. + netsh advfirewall set allprofiles state off + + # Reset IP stack. + netsh winsock reset + + # Disable IPv6. + # https://superuser.com/questions/1545288/disable-ipv6-connectivity-on-windows-without-restarting-the-computer + netsh interface teredo set state disabled + netsh interface ipv6 6to4 set state state=disabled undoonstop=disabled + netsh interface ipv6 isatap set state state=disabled + - name: Invoke tests + shell: bash run: | bin/flake8 - bin/coverage run bin/test -vv1 + + test_program=bin/test + if [ ${{ matrix.os }} = "windows-2019" ]; then + test_program=bin/test-script.py + fi + bin/coverage run $test_program -vv1 + bin/coverage xml # https://github.com/codecov/codecov-action diff --git a/DEVELOP.rst b/DEVELOP.rst index cdd64551..bb2aa992 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -51,7 +51,16 @@ To run against a single interpreter, you can also invoke:: are listening on the default CrateDB transport port to avoid side effects with the test layer. -Preparing a release +In order to run the tests on other operating systems than Linux, invoke:: + + ./bin/test-quick + +This will skip some tests currently not working on macOS. It can also be used +to speed up the tests, because the tests for testing the test layer will also +be skipped. + + +Preparing a Release =================== To create a new release, you must: diff --git a/base.cfg b/base.cfg index e470e47b..24463d3a 100644 --- a/base.cfg +++ b/base.cfg @@ -4,6 +4,7 @@ extends = versions.cfg versions = versions show-picked-versions = true parts = test + test-quick crate scripts coverage @@ -40,6 +41,13 @@ recipe = zc.recipe.testrunner defaults = ['--auto-color'] eggs = crate [test,sqlalchemy] +[test-quick] +relative-paths=true +working-directory = ${buildout:directory} +recipe = zc.recipe.testrunner +defaults = ['--auto-color', '--ignore_dir', 'testing', '--test', '!test_no_retry_on_read_timeout|test_client_threaded|test_client_keepalive|test_username|test_default_schema'] +eggs = crate [test,sqlalchemy] + [linter] recipe = zc.recipe.egg:script eggs = flake8 diff --git a/devtools/setup_ci.sh b/devtools/setup_ci.sh index 6da60665..f564c914 100755 --- a/devtools/setup_ci.sh +++ b/devtools/setup_ci.sh @@ -53,12 +53,14 @@ function main() { # Replace SQLAlchemy version. sed -ir "s/SQLAlchemy.*/SQLAlchemy = ${sqlalchemy_version}/g" versions.cfg - # Replace CrateDB version. - if [ ${cratedb_version} = "nightly" ]; then - sed -ir "s/releases/releases\/nightly/g" base.cfg - sed -ir "s/crate_server.*/crate_server = latest/g" versions.cfg - else - sed -ir "s/crate_server.*/crate_server = ${cratedb_version}/g" versions.cfg + # Adjust CrateDB version, only on Linux. + if [ $(uname -s) = "Linux" ]; then + if [ ${cratedb_version} = "nightly" ]; then + sed -ir "s/releases/releases\/nightly/g" base.cfg + sed -ir "s/crate_server.*/crate_server = latest/g" versions.cfg + else + sed -ir "s/crate_server.*/crate_server = ${cratedb_version}/g" versions.cfg + fi fi buildout -n -c base.cfg diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 7b19a7fb..1ad3f577 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -24,6 +24,8 @@ import json import os import socket +import pathlib +import sys import unittest import doctest from pprint import pprint @@ -166,7 +168,11 @@ def setUpWithCrateLayer(test): cursor.execute(stmt) assert cursor.fetchall()[0][0] == 1 + # Compute path to file for data loading. data_path = docs_path('testing/testdata/data/test_a.json') + if sys.platform == 'win32': + data_path = pathlib.PureWindowsPath(data_path).as_uri() + # load testing data into crate cursor.execute("copy locations from ?", (data_path,)) # refresh location table so imported data is visible immediately diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 3bd3fc99..4b0d9137 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -225,6 +225,9 @@ def __init__(self, self.verbose = verbose self.env = env or {} self.env.setdefault('CRATE_USE_IPV4', 'true') + if sys.platform == 'win32': + self.env.setdefault('_JAVA_OPTIONS', '-Djava.awt.headless=true -Djava.net.preferIPv4Stack=true') + self.env.setdefault('SystemRoot', 'C:\\Windows') self.env.setdefault('JAVA_HOME', os.environ.get('JAVA_HOME', '')) self._stdout_consumers = [] self.conn_pool = urllib3.PoolManager(num_pools=1) @@ -294,6 +297,7 @@ def _clean(self): def start(self): self._clean() + sys.stderr.write("Starting process '{}'\n".format(self.start_cmd)) self.process = subprocess.Popen(self.start_cmd, env=self.env, stdout=subprocess.PIPE) @@ -323,7 +327,17 @@ def start(self): def stop(self): if self.process: self.process.terminate() - self.process.communicate(timeout=10) + try: + self.process.communicate(timeout=10) + except subprocess.TimeoutExpired: + # On GHA/Windows, it always runs into a timeout, even after 45 seconds. + # + # The child process is not killed if the timeout expires, so in order + # to cleanup properly a well-behaved application should kill the child + # process and finish communication. + # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate + self.process.kill() + # self.process.communicate() self.process.stdout.close() self.process = None self.conn_pool.clear() @@ -348,7 +362,7 @@ def _wait_for(self, validator): self.stop() raise e - if wait_time > 30: + if wait_time > 45: for line in line_buf.lines: log.error(line) self.stop() diff --git a/src/crate/testing/tests.py b/src/crate/testing/tests.py index 9c9c3017..1bc39b8a 100644 --- a/src/crate/testing/tests.py +++ b/src/crate/testing/tests.py @@ -28,11 +28,12 @@ def docs_path(*parts): - return os.path.abspath( + path = os.path.abspath( os.path.join( os.path.dirname(os.path.dirname(__file__)), *parts ) ) + return path def project_root(*parts): diff --git a/versions.cfg b/versions.cfg index 2a91d62b..cec6cb50 100644 --- a/versions.cfg +++ b/versions.cfg @@ -20,6 +20,9 @@ py = 1.10.0 tox = 3.23.0 twine = 3.4.0 pkginfo = 1.7.0 +virtualenv = 15.1.0 +wheel = 0.24.0 +zc.buildout = 2.13.7 zc.customdoctests = 1.0.1 zc.recipe.egg = 2.0.7 zc.recipe.testrunner = 2.2