diff --git a/.appveyor.yml b/.appveyor.yml index 40b53bd48a..7e6a91f9ed 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -65,110 +65,6 @@ environment: # List a CI run for each platform first, to have immediate access when there # is a need for debugging - # Ubuntu core tests - - ID: Ubu22core - # ~30min - DTS: > - datalad.cli - datalad.core - datalad.customremotes - datalad.dataset - datalad.distributed - datalad.distribution - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204 - PY: 3.9 - INSTALL_SYSPKGS: python3-virtualenv - CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov - # system git-annex is way too old, use better one - INSTALL_GITANNEX: git-annex -m datalad/packages - # Windows core tests - - ID: WinP39core - # ~35 min - DTS: datalad.core datalad.dataset datalad.runner - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 - # Python version specification is non-standard on windows - PY: 39-x64 - # TODO: use datalad/git-annex (github release packages) but - # it would need setup of a GITHUB_TOKEN to access. - # This one is set in master but kept without change in maint for now - # INSTALL_GITANNEX: git-annex -m datalad/packages - # MacOS core tests - - ID: MacP39core - # ~40min - DTS: datalad.core datalad.dataset datalad.runner datalad.support - APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey - PY: 3.9 - # does not give a functional installation - needs env/PATH adjustment - # INSTALL_GITANNEX: git-annex -m snapshot - #INSTALL_GITANNEX: git-annex=8.20201129 - INSTALL_GITANNEX: git-annex -m datalad/packages - DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets - CODECOV_BINARY: https://cli.codecov.io/v0.7.4/macos/codecov - - # Additional test runs - - ID: Ubu22a1 - # ~30min - DTS: > - datalad.downloaders - datalad.interface - datalad.local - datalad.runner - datalad.support - datalad.tests - datalad.ui - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2204 - PY: 3.9 - INSTALL_SYSPKGS: python3-virtualenv - CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov - # system git-annex is way too old, use better one - INSTALL_GITANNEX: git-annex -m datalad/packages - - ID: WinP39a1 - # ~40min - DTS: > - datalad.cli - datalad.customremotes - datalad.distribution - datalad.distributed - datalad.downloaders - datalad.interface - datalad.tests - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 - PY: 39-x64 - - ID: WinP39a2 - # ~45min - DTS: > - datalad.local - datalad.support - datalad.ui - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 - PY: 39-x64 - - - ID: MacP39a1 - # ~40min - DTS: > - datalad.cli - datalad.customremotes - datalad.distribution - datalad.downloaders - datalad.interface - datalad.tests - datalad.ui - APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey - PY: 3.9 - INSTALL_GITANNEX: git-annex -m datalad/packages - DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets - CODECOV_BINARY: https://cli.codecov.io/v0.7.4/macos/codecov - - ID: MacP39a2 - # ~40min - DTS: > - datalad.local - datalad.distributed - APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey - PY: 3.9 - INSTALL_GITANNEX: git-annex -m datalad/packages - DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets - CODECOV_BINARY: https://cli.codecov.io/v0.7.4/macos/codecov - # Test alternative Python versions - ID: Ubu22P312a # ~35min diff --git a/.github/workflows/add-changelog-snippet.yml b/.github/workflows/add-changelog-snippet.yml deleted file mode 100644 index 0e56efd68d..0000000000 --- a/.github/workflows/add-changelog-snippet.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Add changelog.d snippet - -on: - # This action should be run in workflows triggered by `pull_request_target` - # (not by regular `pull_request`!) - pull_request_target: - # Run whenever the PR is pushed to, receives a label, or is created with - # one or more labels: - types: [synchronize, labeled] - -# Prevent the workflow from running multiple jobs at once when a PR is created -# with multiple labels: -concurrency: - group: ${{ github.workflow }}-${{ github.ref_name }} - cancel-in-progress: true - -jobs: - add: - runs-on: ubuntu-latest - # Only run on PRs that have the "CHANGELOG-missing" label: - if: contains(github.event.pull_request.labels.*.name, 'CHANGELOG-missing') - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.GITMATE_GITHUB_TOKEN }} - - - name: Add changelog snippet - uses: datalad/release-action/add-changelog-snippet@v1 - with: - token: ${{ secrets.GITMATE_GITHUB_TOKEN }} - rm-labels: CHANGELOG-missing diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml deleted file mode 100644 index ac31766159..0000000000 --- a/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Benchmarks - -on: [push, pull_request] - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name }} - cancel-in-progress: true - -jobs: - vs-master: - - runs-on: ubuntu-latest - - steps: - - name: Set up system - shell: bash - run: | - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - sudo apt-get update -qq - sudo apt-get install eatmydata - sudo eatmydata apt-get install git-annex-standalone - - name: Set up environment - run: | - git config --global user.email "test@github.land" - git config --global user.name "GitHub Almighty" - # Prevent interactive credential entry - # See also the GIT_ASKPASS env var below - git config --global core.askPass "" - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install ".[devel-docs]" - # the benchmarks use `ls` - pip install datalad_deprecated - - name: Run benchmarks - env: - # fake environment to be able to reuse script for travis - TRAVIS_PULL_REQUEST: true - # Prevent interactive credential entry (note "true" is the command to run) - GIT_ASKPASS: true - - run: | - tools/ci/benchmark-travis-pr.sh - - name: Compare - run: | - # Invocation from tools/ci/benchmark-travis-pr.sh for convenience - asv compare refs/bm/merge-target refs/bm/pr - - name: Fail if any benchmarks have slowed down too much - run: | - ! asv compare --factor 1.2 --split refs/bm/merge-target refs/bm/pr | grep -q "got worse" diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml deleted file mode 100644 index cd0b7384b8..0000000000 --- a/.github/workflows/docbuild.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Docs - -on: [push, pull_request] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - name: Set up environment - run: | - git config --global user.email "test@github.land" - git config --global user.name "GitHub Almighty" - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install ".[devel-docs]" - sudo apt-get install p7zip - - name: Build docs - run: | - make -C docs html doctest; - - name: Test building manpages - run: | - # with custom date - DATALAD_SOURCE_EPOCH=100000000 python setup.py build_manpage - grep '\.TH "datalad" "1" "1973' ./build/man/datalad.1 - # no custom date - should be good for the next 980 years - python setup.py build_manpage - grep '\.TH "datalad" "1" "2' ./build/man/datalad.1 - - name: Test for correct (unescaped) slashes - run: | - grep '\-\-help' docs/build/html/generated/man/datalad-create.html diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 1e0fc22ae7..0000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Linters - -on: - - push - - pull_request - -jobs: - lint: - runs-on: ubuntu-latest - - steps: - - name: Set up environment - uses: actions/checkout@v4 - with: # no need for the history - fetch-depth: 1 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade tox - - name: Run linters - run: | - tox -e lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 139ebc5945..0000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Auto-release on PR merge - -on: - # This action should be run in workflows triggered by `pull_request_target` - # (not by regular `pull_request`!) - pull_request_target: - branches: - # Create a release whenever a PR is merged into one of these branches: - - maint - types: - - closed - # Allow manually triggering a release via a "Run workflow" button on the - # workflow's page: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - # Only run for manual runs or merged PRs with the "release" label: - if: > - github.event_name == 'workflow_dispatch' - || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release')) - steps: - - name: Checkout source - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITMATE_GITHUB_TOKEN }} - # Check out all history so that the previous release tag can be - # found: - fetch-depth: 0 - - - name: Install pandoc - run: sudo apt-get update && sudo apt-get install -y pandoc - - - name: Prepare release - uses: datalad/release-action/release@v1 - with: - token: ${{ secrets.GITMATE_GITHUB_TOKEN }} - pypi-token: ${{ secrets.PYPI_TOKEN }} - pre-tag: | - make update-changelog - perl -pli -e "s/^version:.*/version: $new_version/" CITATION.cff - git add docs/source/changelog.rst CITATION.cff - git commit -m '[skip ci] Update docs/source/changelog.rst and CITATION.cff' diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml deleted file mode 100644 index bca7506cf4..0000000000 --- a/.github/workflows/shellcheck.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Shellcheck on scripts - -on: [push, pull_request] - -jobs: - test: - - runs-on: ubuntu-latest - - steps: - - name: Set up system - shell: bash - run: | - sudo apt-get update -qq - sudo apt-get install shellcheck - - uses: actions/checkout@v4 - - name: Run shellcheck - run: | - # I: running only on a subset of scripts which are shellcheck clean ATM - shellcheck \ - tools/bisect-git-annex.scripts/bisect-git-annex-lock.sh \ - tools/ci/install-annex.sh \ - tools/ci/install-minimum-git.sh \ - tools/ci/prep-travis-devel-annex.sh \ - tools/ci/prep-travis-forssh.sh diff --git a/.github/workflows/test-label.yml b/.github/workflows/test-label.yml deleted file mode 100644 index 89e5428910..0000000000 --- a/.github/workflows/test-label.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Test for semver label - -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] - -jobs: - test: - runs-on: ubuntu-latest - if: github.repository == 'datalad/datalad' - steps: - - name: Check that PR does not use "major" since we aren't brave enough - run: | - # This should evaluate to either `true` or `false`, which runs the - # command of the same name. - ${{ - !contains(github.event.pull_request.labels.*.name, 'semver-major') - }} - - name: Check that PR against "maint" branch does not have "minor" - if: github.event.pull_request.base.label == 'datalad:maint' - run: | - ${{ - !contains(github.event.pull_request.labels.*.name, 'semver-minor') - }} - - name: Check that PR uses one of the standard "auto" labels - run: | - # major is included to stay optimistic, and to not forget later - ${{ - contains(github.event.pull_request.labels.*.name, 'semver-major') - || contains(github.event.pull_request.labels.*.name, 'semver-minor') - || contains(github.event.pull_request.labels.*.name, 'semver-patch') - || contains(github.event.pull_request.labels.*.name, 'semver-dependencies') - || contains(github.event.pull_request.labels.*.name, 'semver-documentation') - || contains(github.event.pull_request.labels.*.name, 'semver-internal') - || contains(github.event.pull_request.labels.*.name, 'semver-performance') - || contains(github.event.pull_request.labels.*.name, 'semver-tests') - }} - -# vim:set sts=2: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 1a1c148c53..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,247 +0,0 @@ -name: Test - -on: - pull_request: - push: - schedule: - - cron: '0 6 * * *' - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name }} - cancel-in-progress: true - -defaults: - run: - # Use an interactive shell so that ~/.bashrc is sourced: - shell: 'bash -eio pipefail {0}' - -env: - # Note, that there's "turtle" as well, which is always excluded from running. - PYTEST_SELECTION: "integration or usecase or slow or network" - PYTEST_SELECTION_OP: "not " # so it would be "not (integration or usecase)" - DATALAD_TESTS_SSH: "1" - DATALAD_LOG_ENV: GIT_SSH_COMMAND - # How/which git-annex we install. conda's build would be the fastest, - # but it must not get ahead in PATH to not shadow travis' python - _DL_ANNEX_INSTALL_SCENARIO: "miniconda=py37_23.1.0-1 --python-match minor --batch git-annex=8.20201007 -m conda" - BOTO_CONFIG: /tmp/nowhere - DATALAD_DATASETS_TOPURL: https://datasets-tests.datalad.org - -jobs: - filter: - runs-on: ubuntu-latest - outputs: - jobs: ${{ steps.jobs.outputs.matrix }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Possibly filter out cron-only jobs - id: jobs - run: | - echo 'matrix<> "$GITHUB_OUTPUT" - if [ "${{ github.event_name == 'schedule' }}" = true ] - then - yq '{"include": .} | to_json' tools/ci/test-jobs.yml >> "$GITHUB_OUTPUT" - else - yq '{"include": [.[] | select(."cron-only" != true)]} | to_json' tools/ci/test-jobs.yml >> "$GITHUB_OUTPUT" - fi - echo 'EOT' >> "$GITHUB_OUTPUT" - - test: - runs-on: ubuntu-latest - needs: filter - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.filter.outputs.jobs) }} - env: ${{ matrix.extra-envs }} - continue-on-error: ${{ matrix.allow-failure || false }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - # Do full clone (~10 extra seconds) to fetch all tags. - # Otherwise we might be missing the tags for maint PRs - # whenever those maint releases were not yet merged into master. - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Show git describe output to ensure that we did fetch all the tags - run: git describe - - # Just in case we need to check if nfs is there etc - - run: sudo lsmod - - - name: Install dependencies - run: | - # The ultimate one-liner setup for NeuroDebian repository - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - sudo apt-get update -qq - sudo apt-get install eatmydata # to speedup some installations - tools/ci/prep-travis-forssh.sh - tools/ci/debians_disable_outdated_ssl_cert - # Install various basic dependencies - sudo eatmydata apt-get install zip pandoc p7zip-full - # needed for tests of patool compression fall-back solution - sudo eatmydata apt-get install xz-utils - sudo eatmydata apt-get install shunit2 - [ -z "$LC_ALL" ] || sudo eatmydata apt-get install locales-all - - - name: Configure _DL_TMPDIR before installing git-annex - run: | - if [[ "${_DL_TMPDIR:-}" =~ .*/sym\ link ]] - then echo "Symlinking $_DL_TMPDIR" - ln -s /tmp "$_DL_TMPDIR" - fi - - if [[ "${_DL_TMPDIR:-}" =~ .*/d\ i\ r ]] - then echo "mkdir $_DL_TMPDIR" - mkdir -p "$_DL_TMPDIR" - fi - - if [[ "${_DL_TMPDIR:-}" =~ .*/nfsmount ]] - then echo "mkdir $_DL_TMPDIR" - mkdir -p "$_DL_TMPDIR" "${_DL_TMPDIR}_" - echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports' - sudo apt-get install -y nfs-kernel-server - sudo exportfs -a - sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount - fi - - - name: Install custom Git from upstream - if: matrix.upstream-git - run: | - sudo apt-get install -y gettext libcurl4-gnutls-dev - source tools/ci/install-upstream-git.sh - echo "$target_dir/bin-wrappers" >> "$GITHUB_PATH" - - - name: Install minimum Git - if: matrix.minimum-git - run: | - sudo apt-get install -y gettext libcurl4-gnutls-dev - tools/ci/install-minimum-git.sh - echo "$PWD/git-src/bin-wrappers" >> "$GITHUB_PATH" - - - name: Install git-annex - run: | - pip install datalad-installer - eval datalad-installer --sudo ok -E new.env ${_DL_ANNEX_INSTALL_SCENARIO} - # Append new.env to ~/.bashrc instead of $GITHUB_OUTPUT because it - # can include `source` and `conda activate` commands that are invalid - # for the latter; this also necessitates using an interactive shell - # for the workflow. - cat new.env >> ~/.bashrc - - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install codecov - pip install -r requirements-devel.txt - - - name: Configure Git - run: | - git config --global user.email "test@github.land" - git config --global user.name "GitHub Almighty" - # We do `sudo pip install` below, and versioneer needs to run git. - # Recent git needs to be made certain it is safe to do - sudo git config --global --add safe.directory $PWD - - - name: Configure sudoers - # So we could test under sudo -E with PATH pointing to installed - # location - run: sudo sed -i -e 's/^Defaults.*secure_path.*$//' /etc/sudoers - - # TODO: remove - should not be needed - - name: Git-annex workaround for NFS mounts - run: | - if [[ "${_DL_TMPDIR:-}" =~ .*/nfsmount ]] - then sudo git config --system annex.pidlock true - fi - - # Now it should be safe to point TMPDIR to a "tricky" setup just for the - # purpose of testing - - name: Point TMPDIR to "tricky" setup - run: | - if [ -n "${_DL_TMPDIR:-}" ] - then echo TMPDIR="${_DL_TMPDIR}" >> "$GITHUB_ENV" - fi - - - name: Test installation for user - run: sudo pip install --user . - - - name: Report WTF information using system-wide installed version - run: | - datalad wtf - python -m pip freeze - - - name: Run tests - run: | - PYTEST_OPTS=( -v ) - # If we requested to run only not slow (typically <10sec) tests, fail - # if a test takes 3x more than that - it needs to get @slow or - # @turtle annotation - if echo "$PYTEST_SELECTION_OP($PYTEST_SELECTION)" | grep -q "^not.*slow" - then - PYTEST_OPTS=( "${PYTEST_OPTS[@]}" --doctest-modules --durations=0 --durations-min=5 --fail-slow 60 ) - export DATALAD_TESTS_SETUP_TESTREPOS=1 - fi - - mkdir -p __testhome__ - cd __testhome__ - # Note: adding --log-cli-level=INFO would result in - # DATALAD_LOG_TARGET=/dev/null being not in effect, dumping too many - # logs. - set -x - http_proxy= - PATH=$PWD/../tools/coverage-bin:$PATH - $PYTEST_WRAPPER python \ - -m pytest "${PYTEST_OPTS[@]}" \ - -c ../tox.ini \ - -n 2 \ - -m "${PYTEST_SELECTION:+$PYTEST_SELECTION_OP($PYTEST_SELECTION) and }not(turtle)" \ - --doctest-modules \ - --cov=datalad \ - --cov-report=xml \ - --pyargs ${TESTS_TO_PERFORM:-datalad} - - # Makes it only more difficult to comprehend the failing output. Enable - # only when necessary for a particular debugging. - - name: Dump debug info - #if: "failure()" # Enabled - if: "failure() && false" # Disabled - run: | - if [ ! -z "$DATALAD_TESTS_NONETWORK" ] - then sudo route add -net 0.0.0.0 netmask 0.0.0.0 dev lo - fi - - DATALAD_LOG_LEVEL=DEBUG \ - $PYTEST_WRAPPER `which pytest` \ - -s -v --doctest-modules --cov datalad --log-cli-level=DEBUG - - if [ ! -z "$DATALAD_TESTS_NONETWORK" ] - then sudo route del -net 0.0.0.0 netmask 0.0.0.0 dev lo - fi - - # cron jobs test more and then PRs will be falling behind since they - # would not trigger some codepaths. So submit coverage only from - # non-cron jobs, but report for all - - - name: Report coverage - run: | - python -m coverage debug sys - python -m coverage report - working-directory: __testhome__ - - - name: Upload coverage to Codecov - if: github.event_name != 'schedule' - uses: codecov/codecov-action@v5 - with: - directory: __testhome__ - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} - -# vim:set et sts=2: diff --git a/.github/workflows/test_crippled.yml b/.github/workflows/test_crippled.yml deleted file mode 100644 index c68566b3bb..0000000000 --- a/.github/workflows/test_crippled.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: CrippledFS - -on: [pull_request] - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name }} - cancel-in-progress: true - -jobs: - test: - - runs-on: ubuntu-latest - steps: - - name: Set up system - shell: bash - run: | - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - # enable repo for devel git-annex if needed - # sudo sed -e 's|\(deb.*data\)|#\1|' -e 's|/debian |/debian-devel |' /etc/apt/sources.list.d/neurodebian.sources.list | sudo tee /etc/apt/sources.list.d/neurodebian-devel.sources.list - sudo apt-get update -qq - sudo apt-get install eatmydata - sudo eatmydata apt-get install git-annex-standalone dosfstools - # 500 MB VFAT FS in a box - sudo dd if=/dev/zero of=/crippledfs.img count=500 bs=1M - sudo mkfs.vfat /crippledfs.img - # mount - sudo mkdir /crippledfs - sudo mount -o "uid=$(id -u),gid=$(id -g)" /crippledfs.img /crippledfs - - name: Set up environment - run: | - git config --global user.email "test@github.land" - git config --global user.name "GitHub Almighty" - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: WTF!? - run: | - datalad wtf - mount - - name: Run tests - env: - # forces all test repos/paths into the VFAT FS - TMPDIR: /crippledfs - run: | - mkdir -p __testhome__ - cd __testhome__ - python -m pytest -c ../tox.ini -s -v -m "not (turtle)" --pyargs datalad.core datalad.local datalad.distributed datalad.tests.test_utils_cached_dataset diff --git a/.github/workflows/test_extensions.yml b/.github/workflows/test_extensions.yml deleted file mode 100644 index 164b1248a9..0000000000 --- a/.github/workflows/test_extensions.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Extensions - -on: [push, pull_request] - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name }} - cancel-in-progress: true - -jobs: - test: - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - extension: [ - datalad-neuroimaging, - datalad-container, - datalad-metalad, - datalad-crawler, - datalad-deprecated, - datalad-next, - ] - - steps: - - name: Set up system - shell: bash - run: | - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - # enable in case -devel version is needed - # sudo sed -i-devel.list -e 's,/debian ,/debian-devel ,g' /etc/apt/sources.list.d/neurodebian.sources.list - sudo apt-get update -qq - sudo apt-get install eatmydata - sudo eatmydata apt-get install git-annex-standalone - - - name: Set up environment - run: | - git config --global user.email "test@github.land" - git config --global user.name "GitHub Almighty" - - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - - name: Install DataLad and dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-devel.txt - - - name: Setup ${{ matrix.extension }} extension specific variables - run: | - DL_PIP_INSTALLS=mock - DL_APT_INSTALLS=dcm2niix - DL_REPO=datalad/${{ matrix.extension }} - DL_TESTER=pytest - DL_NEED_SINGULARITY= - # TODO: just reuse information from datalad-extensions!! - case ${{ matrix.extension }} in - datalad-container) - DL_NEED_SINGULARITY=1;; - esac - { - echo "DL_PIP_INSTALLS=$DL_PIP_INSTALLS" - echo "DL_APT_INSTALLS=$DL_APT_INSTALLS" - echo "DL_REPO=$DL_REPO" - echo "DL_TESTER=$DL_TESTER" - echo "DL_NEED_SINGULARITY=$DL_NEED_SINGULARITY" - echo "DL_PACKAGE=$(echo ${{ matrix.extension }} | tr '-' '_')" - } >> "$GITHUB_ENV" - - - name: Clone and install ${{ matrix.extension }} extension - run: | - git clone https://github.com/$DL_REPO __extension__ - pip install ./__extension__[devel] - - - name: Install Singularity - run: tools/ci/install-singularity.sh - if: env.DL_NEED_SINGULARITY == '1' - - - name: Install additional dependencies and prepare for testing - run: | - pip install $DL_PIP_INSTALLS - [ -z "$DL_APT_INSTALLS" ] || sudo apt-get install $DL_APT_INSTALLS - mkdir -p __testhome__ - - - name: WTF!? - run: | - datalad wtf - python -m pip freeze - - - name: ${{ matrix.extension }} tests using pytest - run: | - cd __testhome__ - # -crawler's tox.ini points to itself (but without full path) for coverage - # configuration. So far attempts to "fix" by somehow providing full path did not work. - # That is why just symlink it here. If no tox.ini -- create an empty one - [ -e ../__extension__/tox.ini ] && ln -s ../__extension__/tox.ini . || touch tox.ini - python -m pytest -c ./tox.ini -s -v --cov=datalad --pyargs ${DL_PACKAGE} - if: env.DL_TESTER == 'pytest' - - - name: Prepare coverage - run: | - cd __testhome__ - python -m coverage debug sys - python -m coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - directory: __testhome__ - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml deleted file mode 100644 index 1352a725de..0000000000 --- a/.github/workflows/typing.yml +++ /dev/null @@ -1,32 +0,0 @@ -# This workflow is running types checks using `tox -e typing` you could do locally. -# As type hinting is not yet fully and consistently done through out the project only -# some (randomly) selected files (listed in tox.ini) will be checked. Feel welcome -# to add extra files to the list there. See https://github.com/datalad/datalad/issues/6884 -# for the overall goal/progress towards type hints coverage in DataLad. -name: Type-check - -on: - - push - - pull_request - -jobs: - typing: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade tox - - - name: Run type checker - run: tox -e typing diff --git a/.github/workflows/update-contributors.yml b/.github/workflows/update-contributors.yml deleted file mode 100644 index 4d70b08314..0000000000 --- a/.github/workflows/update-contributors.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: allcontributors-auto-detect - -on: - push: - branches: - - master - -jobs: - Update: - name: Generate - runs-on: ubuntu-latest - if: contains(github.repository, 'datalad/datalad') - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Tributors Update - uses: con/tributors@0.1.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - - # Single text list (space separated) of parsers, leave unset to auto-detect - parsers: unset - - # Update lookup with GitHub metadata - update_lookup: github - - # Skip these users (example) - skip_users: manbat bhanuprasad14 yetanothertestuser bobknob23987 - - # INFO, DEBUG, ERROR, WARNING, etc. - log_level: DEBUG - - # If files already exist and an init is done, force overwrite - force: true - - # the minimum number of contributions required to add a user - threshold: 1 - - - name: Checkout New Branch - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH_AGAINST: "master" - run: | - printf "GitHub Actor: ${GITHUB_ACTOR}\n" - export BRANCH_FROM="contributors/update-$(date '+%Y-%m-%d')" - git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - - BRANCH_EXISTS=$(git ls-remote --heads origin ${BRANCH_FROM}) - if [[ -z ${BRANCH_EXISTS} ]]; then - printf "Branch does not exist in remote.\n" - else - printf "Branch already exists in remote.\n" - exit 1 - fi - git branch - git checkout -b "${BRANCH_FROM}" || git checkout "${BRANCH_FROM}" - git branch - - git config --global user.name "github-actions" - git config --global user.email "github-actions@users.noreply.github.com" - git status - - if git diff-index --quiet HEAD --; then - export OPEN_PULL_REQUEST=0 - printf "No changes\n" - else - export OPEN_PULL_REQUEST=1 - printf "Changes\n" - git commit -a -m "Automated deployment to update contributors $(date '+%Y-%m-%d') - - [skip ci]" - git push origin "${BRANCH_FROM}" - fi - echo "OPEN_PULL_REQUEST=${OPEN_PULL_REQUEST}" >> $GITHUB_ENV - echo "PULL_REQUEST_FROM_BRANCH=${BRANCH_FROM}" >> $GITHUB_ENV - echo "PULL_REQUEST_TITLE=[tributors] ${BRANCH_FROM}" >> $GITHUB_ENV - echo "PULL_REQUEST_BODY='Tributors update automated pull request.\n\n[skip ci]'" >> $GITHUB_ENV - - - name: Open Pull Request - uses: vsoch/pull-request-action@1.1.1 - if: ${{ env.OPEN_PULL_REQUEST == '1' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PULL_REQUEST_BRANCH: "master"