diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b2fbe8f..16370a0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,95 +2,137 @@ name: packages on: push: tags: - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+a[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+b[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+rc[0-9]+' + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+a[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+b[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" # Dry-run only workflow_dispatch: +defaults: + run: + shell: bash -l {0} + +env: + PYTHON_VERSION: "3.10" + PACKAGE: "intake-dremio" + jobs: + waiting_room: + name: Waiting Room + runs-on: ubuntu-latest + needs: [conda_build, pip_install] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - run: echo "All builds have finished, have been approved, and ready to publish" + conda_build: - name: Build Conda Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} - env: - CHANS_DEV: "-c conda-forge" - PKG_TEST_PYTHON: "--test-python=py37" - PYTHON_VERSION: "3.7" - CHANS: "-c pyviz" + name: Build Conda + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Fetch unshallow run: git fetch --prune --tags --unshallow -f - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - python-version: 3.8 - - name: Set output - id: vars - run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} + python-version: ${{ env.PYTHON_VERSION }} - name: conda setup run: | - conda config --set always_yes True - conda install -c pyviz "pyctdev>=0.5" - doit ecosystem_setup + conda install -y conda-build python-build - name: conda build - run: doit package_build $CHANS_DEV $PKG_TEST_PYTHON --test-group=all - - name: conda dev upload - if: (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')) - run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev - - name: conda main upload - if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) - run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main - pip_build: - name: Build PyPI Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} - env: - CHANS_DEV: "-c pyviz/label/dev -c conda-forge" - PKG_TEST_PYTHON: "--test-python=py37" - PYTHON_VERSION: "3.7" - CHANS: "-c pyviz" - PPU: ${{ secrets.PPU }} - PPP: - PYPI: "https://upload.pypi.org/legacy/" + run: | + source ./scripts/conda-build.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: conda + path: dist/*.tar.bz2 + if-no-files-found: error + + conda_publish: + name: Publish Conda + runs-on: ubuntu-latest + needs: [conda_build, waiting_room] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/checkout@v2 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - uses: actions/setup-python@v2 + - uses: actions/download-artifact@v4 with: - python-version: ${{ matrix.python-version }} - - uses: conda-incubator/setup-miniconda@v2 + name: conda + path: dist/ + - name: Set environment variables + run: | + echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV + - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - name: conda setup run: | - conda config --set always_yes True - conda install -c pyviz "pyctdev>=0.5" - doit ecosystem_setup - doit env_create $CHANS_DEV --python=$PYTHON_VERSION - - name: env setup + conda install -y anaconda-client + - name: conda dev upload + if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') run: | - eval "$(conda shell.bash hook)" - conda activate test-environment - doit develop_install $CHANS_DEV -o tests - doit pip_on_conda - - name: pip build + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE + - name: conda main upload + if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) run: | - eval "$(conda shell.bash hook)" - conda activate test-environment - doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit_deploy - - name: pip upload + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + + pip_build: + name: Build PyPI + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v4 + - name: Fetch unshallow + run: git fetch --prune --tags --unshallow -f + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install build run: | - eval "$(conda shell.bash hook)" - conda activate test-environment - doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI + python -m pip install build + - name: Build package + run: python -m build . + - uses: actions/upload-artifact@v4 + if: always() + with: + name: pip + path: dist/ + if-no-files-found: error + + pip_install: + name: Install PyPI + runs-on: "ubuntu-latest" + needs: [pip_build] + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Install package + run: python -m pip install dist/*.whl + - name: Test package + env: + PACKAGE: "intake_dremio" + run: python -c "import $PACKAGE; print($PACKAGE.__version__)" + + pip_publish: + name: Publish PyPI + runs-on: ubuntu-latest + needs: [pip_build, waiting_room] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: ${{ secrets.PPU }} + password: ${{ secrets.PPP }} + repository-url: "https://upload.pypi.org/legacy/" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fd0d91b..97e7904 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,22 +16,21 @@ jobs: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: [3.6, 3.7, 3.8] + python-version: ["3.9", "3.10", "3.11"] timeout-minutes: 30 defaults: run: shell: bash -l {0} env: PYTHON_VERSION: ${{ matrix.python-version }} - CHANS_DEV: "-c conda-forge" steps: - uses: actions/checkout@v2 with: fetch-depth: "100" - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - name: Fetch @@ -44,27 +43,22 @@ jobs: doit env_create ${{ env.CHANS_DEV}} --python=${{ matrix.python-version }} - name: doit develop_install run: | - eval "$(conda shell.bash hook)" conda activate test-environment conda list doit develop_install ${{ env.CHANS_DEV }} -o tests - name: doit env_capture run: | - eval "$(conda shell.bash hook)" conda activate test-environment doit env_capture - name: doit test_lint run: | - eval "$(conda shell.bash hook)" conda activate test-environment doit test_lint - name: doit test_unit run: | - eval "$(conda shell.bash hook)" conda activate test-environment doit test_unit_deploy - name: codecov run: | - eval "$(conda shell.bash hook)" conda activate test-environment codecov diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 1641191..cbff074 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,15 +1,15 @@ -{% set sdata = load_setup_py_data() %} +{% set sdata = load_setup_py_data(setup_file="../setup.py", from_recipe_dir=True) %} package: name: intake-dremio version: {{ sdata['version'] }} source: - path: .. + url: ../dist/{{ sdata['name'].replace('-', '_') }}-{{ VERSION }}-py3-none-any.whl build: noarch: python - script: python setup.py install --single-version-externally-managed --record=record.txt + script: {{ PYTHON }} -m pip install -vv {{ sdata['name'].replace('-', '_') }}-{{ VERSION }}-py3-none-any.whl entry_points: {% for group,epoints in sdata.get("entry_points", {}).items() %} {% for entry_point in epoints %} diff --git a/intake_dremio/intake_dremio.py b/intake_dremio/intake_dremio.py index 9755645..7d924aa 100644 --- a/intake_dremio/intake_dremio.py +++ b/intake_dremio/intake_dremio.py @@ -1,3 +1,5 @@ +from functools import reduce + from intake.source import base from pyarrow import flight @@ -38,13 +40,23 @@ def __init__(self, factory): self.factory = factory def received_headers(self, headers): - auth_header_key = 'authorization' - authorization_header = [] - for key in headers: - if key.lower() == auth_header_key: - authorization_header = headers.get(auth_header_key) - self.factory.set_call_credential([ - b'authorization', authorization_header[0].encode("utf-8")]) + if self.factory.call_credential: + return + + auth_header_key = "authorization" + + authorization_header = reduce( + lambda result, header: ( + header[1] if header[0] == auth_header_key else result + ), + headers.items(), + ) + if not authorization_header: + raise Exception("Did not receive authorization header back from server.") + bearer_token = authorization_header[1][0] + self.factory.set_call_credential( + [b"authorization", bearer_token.encode("utf-8")] + ) class DremioClientAuthMiddlewareFactory(flight.ClientMiddlewareFactory): diff --git a/intake_dremio/tests/test_intake_dremio.py b/intake_dremio/tests/test_intake_dremio.py index fc33fae..efb68b2 100644 --- a/intake_dremio/tests/test_intake_dremio.py +++ b/intake_dremio/tests/test_intake_dremio.py @@ -3,7 +3,7 @@ from intake_dremio import DremioSource # pytest imports this package last, so plugin is not auto-added -intake.registry['dremio'] = DremioSource +intake.source.discovery.drivers.register_driver('dremio', DremioSource, clobber=True) def test_constructor(): source = DremioSource('user:password@localhost:32010', 'SELECT * FROM table') diff --git a/scripts/conda-build.sh b/scripts/conda-build.sh new file mode 100755 index 0000000..f6c93a2 --- /dev/null +++ b/scripts/conda-build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +PACKAGE="intake-dremio" + +for file in dist/*.whl dist/*.tar.bz2; do + if [ -e "$file" ]; then + echo "dist folder already contains $(basename "$file"). Please delete it before running this script." + exit 1 + fi +done + +git diff --exit-code +python -m build -w . + +VERSION=$(find dist -name "*.whl" -exec basename {} \; | cut -d- -f2) +export VERSION + +conda build conda.recipe --no-anaconda-upload --no-verify + +mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2" dist diff --git a/setup.py b/setup.py index 0ce2ef7..093380a 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ install_requires = [ 'pandas >=0.25', - 'intake', + 'intake <2', 'pyarrow >=4.0' ] diff --git a/tox.ini b/tox.ini index 702448e..beaed09 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ [tox] # python version test group extra envs extra commands -envlist = {py36,py37,py38}-{lint,unit,unit_deploy,all}-{default}-{dev,pkg} +envlist = {py39,py310,py311,py312}-{lint,unit,unit_deploy,all}-{default}-{dev,pkg} build = wheel [_lint]