diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..8059890 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,4 @@ +- [ ] I have read the [section on commits and pull requests](https://github.com/NaturalHistoryMuseum/ckanext-statistics/blob/main/CONTRIBUTING.md#commits-and-pull-requests) in `CONTRIBUTING.md` + + +Describe your changes, tagging relevant issues where possible. diff --git a/.github/workflows/bump.yml b/.github/workflows/bump.yml index 8b17261..1bb1860 100644 --- a/.github/workflows/bump.yml +++ b/.github/workflows/bump.yml @@ -7,12 +7,12 @@ on: jobs: bump-version: - if: "!startsWith(github.event.head_commit.message, 'bump:')" + name: Bump version and create changelog runs-on: ubuntu-latest - name: "Bump version and create changelog" + if: "!startsWith(github.event.head_commit.message, 'bump:')" steps: - - name: Check out - uses: actions/checkout@v3 + - name: Checkout source code + uses: actions/checkout@v4 with: token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} fetch-depth: 0 diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml new file mode 100644 index 0000000..134e108 --- /dev/null +++ b/.github/workflows/pull-requests.yml @@ -0,0 +1,27 @@ +name: Validate pull requests + +on: + pull_request: + types: [opened, edited, reopened, synchronize] + +jobs: + validate-commits: + name: Validate commit messages + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Check commit message format + uses: webiny/action-conventional-commits@v1.3.0 + with: + allowed-commit-types: 'feat,fix,refactor,perf,docs,style,test,build,ci,chore,new,patch,revert,ui,merge' + pre-commit: + name: Run pre-commit checks + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 6866a49..bf30290 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -10,17 +10,16 @@ permissions: jobs: deploy: + name: Deploy package to PyPI runs-on: ubuntu-latest steps: - - name: Check out - uses: actions/checkout@v3 + - name: Checkout source code + uses: actions/checkout@v4 with: token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' + uses: actions/setup-python@v5 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index bc66f38..4fec115 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -7,11 +7,11 @@ on: jobs: sync-branches: + name: Sync dev and patch branches to latest commit runs-on: ubuntu-latest - name: "Sync dev and patch branches to latest commit" steps: - - name: Check out - uses: actions/checkout@v3 + - name: Checkout source code + uses: actions/checkout@v4 with: token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} fetch-depth: 0 diff --git a/.github/workflows/main.yml b/.github/workflows/tests.yml similarity index 77% rename from .github/workflows/main.yml rename to .github/workflows/tests.yml index 504d4d5..9350b72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/tests.yml @@ -3,18 +3,18 @@ name: Tests on: push: workflow_dispatch: + pull_request: + types: [opened, edited, reopened, synchronize] jobs: test: + name: Run tests runs-on: ubuntu-latest - steps: - name: Checkout source code - uses: actions/checkout@v3 - + uses: actions/checkout@v4 - name: Build images run: docker compose build - - name: Run tests env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d3b017..04e9d79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,33 +1,34 @@ exclude: /(vendor|dist)/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: detect-private-key - id: end-of-file-fixer - id: name-tests-test - args: ["--pytest-test-first"] + args: ['--pytest-test-first'] exclude: ^tests/helpers/ - id: trailing-whitespace - repo: https://github.com/commitizen-tools/commitizen - rev: v2.37.0 + rev: v3.30.0 hooks: - id: commitizen - additional_dependencies: ["cz-nhm"] - - repo: https://github.com/psf/black - rev: 22.10.0 + additional_dependencies: ['cz-nhm'] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.1 hooks: - - id: black + - id: ruff + args: [ '--fix', '--select', 'I', '--select', 'F401', '--fix-only' ] + - id: ruff-format - repo: https://github.com/PyCQA/docformatter - rev: v1.5.0 + rev: eb1df34 hooks: - id: docformatter - # these can't be pulled directly from the config atm, not sure why - args: ["-i", "--wrap-summaries=88", "--wrap-descriptions=88", - "--pre-summary-newline", "--make-summary-multi-line"] + args: [ '-i', '--config', './pyproject.toml' ] + additional_dependencies: ['tomli'] - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.4 + rev: v4.0.0-alpha.8 hooks: - id: prettier types_or: [ javascript, vue, less, sass, scss, css ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a9fd6a..d90990c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ This extension and [several others](https://github.com/search?q=topic:ckan+org:N The current core team consists of: - Josh ([@jrdh](https://github.com/jrdh)) - Technical Lead -- Ginger ([@alycejenni](https://github.com/alycejenni)) - Software Engineer +- Ginger ([@alycejenni](https://github.com/alycejenni)) - Senior Software Engineer ## Questions @@ -81,7 +81,10 @@ The process is generally as follows: 3. Make your changes, and commit often; each commit should only contain one change. See below for specifics on how to word your commits. 4. Push your changes back to your fork. 5. [Open a pull request](https://docs.github.com/en/get-started/quickstart/contributing-to-projects#making-a-pull-request) in this repository, with the base branch set to **dev** and the compare branch set to your new branch. Provide a summary of your changes in the description. -6. If the automatic tests fail (these may take a while), please go back to your code and try to make them pass. You may have to update the tests themselves. You don't have to close the pull request while you're doing this; it'll update as you add further commits. +6. There are several automated checks that will run when you open the pull request. Try to make all of them pass. If you do not at least _attempt_ to make them pass, we will not merge your pull request. + 1. Tests. Update them so that they pass, if necessary. New tests are always welcome in any pull request, but if you have added a new feature that has decreased the coverage, new tests are required. + 2. Commit format validation. If you have not followed the conventional commits format for one or more of your commits, this will fail. + 3. Code format validation. If you have not formatted your code correctly (using Ruff, docformatter, and/or Prettier), this will fail. 7. Wait for feedback from one of the core maintainers. If it's been a week or so and we haven't responded, we may not have seen it. You can find other places to contact us in [SUPPORT.md](./.github/SUPPORT.md). ### Commits @@ -142,9 +145,9 @@ cz c ##### pre-commit -pre-commit is a tool that runs a variety of checks and modifications before a commit is made. You can check the [.pre-commit-config.yaml](./.pre-commit-config.yaml) file to see eaxtly what it's currently configured to do for this repository, but of particular note: +pre-commit is a tool that runs a variety of checks and modifications before a commit is made. You can check the [.pre-commit-config.yaml](./.pre-commit-config.yaml) file to see exactly what it's currently configured to do for this repository, but of particular note: -- reformats Python code with [Black](https://github.com/psf/black) +- reformats Python code with [Ruff](https://docs.astral.sh/ruff) - reformats JavaScript and stylesheets with [Prettier](https://prettier.io) - reformats docstrings with [docformatter](https://github.com/PyCQA/docformatter) - checks your commit message is correcly formatted @@ -161,15 +164,15 @@ pre-commit run Don't forget to stage any modifications that it makes! Once it runs without failing, then you can make your commit. -Something to remember is that empty docstrings will cause conflicts between Black and docformatter and the checks will fail repeatedly - so don't leave your docstrings empty! +Something to remember is that empty docstrings will cause conflicts between Ruff and docformatter and the checks will fail repeatedly - so don't leave your docstrings empty! ### Code changes and style guide -We generally use external style guides and tools to help us maintain standardised code. Black and Prettier will be run with pre-commit. +We generally use external style guides and tools to help us maintain standardised code. Ruff and Prettier will be run with pre-commit. #### Python -We follow the [Black style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html), with the notable exception that we use single quotes. +We use [Ruff](https://docs.astral.sh/ruff) to format our code, using defaults for everything except quote style (we use single quotes). We also _mostly_ use [CKAN's style](http://docs.ckan.org/en/latest/contributing/python.html), with the following exceptions: - prefer `f''` strings over `.format()` @@ -178,7 +181,7 @@ We also _mostly_ use [CKAN's style](http://docs.ckan.org/en/latest/contributing/ #### JavaScript and stylesheets (CSS, LESS, etc) -We use [Prettier](https://prettier.io) to format these files. +We use [Prettier](https://prettier.io) to format these files. As with Ruff, we use defaults for everything except quote style (we use single quotes). #### Accessibility diff --git a/README.md b/README.md index e590d7b..694493e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ - + + + + The Natural History Museum logo. + # ckanext-statistics -[![Tests](https://img.shields.io/github/actions/workflow/status/NaturalHistoryMuseum/ckanext-statistics/main.yml?style=flat-square)](https://github.com/NaturalHistoryMuseum/ckanext-statistics/actions/workflows/main.yml) +[![Tests](https://img.shields.io/github/actions/workflow/status/NaturalHistoryMuseum/ckanext-statistics/tests.yml?style=flat-square)](https://github.com/NaturalHistoryMuseum/ckanext-statistics/actions/workflows/tests.yml) [![Coveralls](https://img.shields.io/coveralls/github/NaturalHistoryMuseum/ckanext-statistics/main?style=flat-square)](https://coveralls.io/github/NaturalHistoryMuseum/ckanext-statistics) [![CKAN](https://img.shields.io/badge/ckan-2.9.7-orange.svg?style=flat-square)](https://github.com/ckan/ckan) [![Python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg?style=flat-square)](https://www.python.org/) diff --git a/ckanext/statistics/cli.py b/ckanext/statistics/cli.py index cfb3d13..a3c0baa 100644 --- a/ckanext/statistics/cli.py +++ b/ckanext/statistics/cli.py @@ -4,7 +4,7 @@ from ckan.plugins import toolkit from tqdm import tqdm -from .model.gbif_download import gbif_downloads_table, GBIFDownload +from .model.gbif_download import GBIFDownload, gbif_downloads_table def get_commands(): @@ -79,7 +79,7 @@ def get_gbif_stats(dataset_uuid, limit=100): :param dataset_uuid: the dataset uuid to get stats records for :param limit: the number of records to retrieve at once (default: 100) - :return: yields dicts + :returns: yields dicts """ offset = 0 url = f'https://api.gbif.org/v1/occurrence/download/dataset/{dataset_uuid}' diff --git a/ckanext/statistics/lib/dataset_statistics.py b/ckanext/statistics/lib/dataset_statistics.py index 5c2f9e8..b9737a6 100644 --- a/ckanext/statistics/lib/dataset_statistics.py +++ b/ckanext/statistics/lib/dataset_statistics.py @@ -6,6 +6,7 @@ import logging + from ckan.lib.search import SearchIndexError from ckan.plugins import toolkit diff --git a/ckanext/statistics/lib/download_statistics.py b/ckanext/statistics/lib/download_statistics.py index e9914cd..f5650d3 100644 --- a/ckanext/statistics/lib/download_statistics.py +++ b/ckanext/statistics/lib/download_statistics.py @@ -8,13 +8,14 @@ import json from collections import OrderedDict, defaultdict +import ckan.model as model +from ckan.plugins import toolkit from importlib_resources import files from sqlalchemy import sql -import ckan.model as model -from ckan.plugins import toolkit from ckanext.ckanpackager.model.stat import CKANPackagerStat from ckanext.versioned_datastore.model.downloads import DownloadRequest + from ..lib.statistics import Statistics from ..logic.schema import statistics_downloads_schema from ..model.gbif_download import GBIFDownload @@ -28,11 +29,11 @@ class MonthlyStats(object): """ def __init__(self, month=None, year=None, resource_id=None): - ''' + """ :param month: if passed, only this month will be counted (defaults to None) :param year: if passed, only this year will be counted (defaults to None) :param resource_id: if passed, only this resource will be counted (defaults to None) - ''' + """ self.month = int(month) if month is not None else None self.year = int(year) if year is not None else None self.resource_id = resource_id @@ -117,8 +118,8 @@ def update_from_backfill(self, month, year, stats): :param month: the month :param year: the year - :param stats: the stats in a dict, the format of this dict must match a month/year entry in - the self.stats dict + :param stats: the stats in a dict, the format of this dict must match a + month/year entry in the self.stats dict """ if self.month is not None and self.month != int(month): return @@ -158,7 +159,7 @@ def as_dict(self): """ Return an OrderedDict of count stats in ascending chronological order. - :return: an OrderedDict + :returns: an OrderedDict """ return OrderedDict( sorted( diff --git a/ckanext/statistics/model/gbif_download.py b/ckanext/statistics/model/gbif_download.py index a032507..4ebbb02 100644 --- a/ckanext/statistics/model/gbif_download.py +++ b/ckanext/statistics/model/gbif_download.py @@ -5,8 +5,8 @@ # Created by the Natural History Museum in London, UK -from ckan.model import meta, DomainObject -from sqlalchemy import Column, DateTime, Integer, func, Table, UnicodeText +from ckan.model import DomainObject, meta +from sqlalchemy import Column, DateTime, Integer, Table, UnicodeText, func gbif_downloads_table = Table( 'gbif_downloads', diff --git a/ckanext/statistics/plugin.py b/ckanext/statistics/plugin.py index 64fb5d5..d82caef 100644 --- a/ckanext/statistics/plugin.py +++ b/ckanext/statistics/plugin.py @@ -8,6 +8,7 @@ from ckan.plugins import SingletonPlugin, implements, interfaces from ckanext.statistics.logic.action import dataset_statistics, download_statistics + from . import cli diff --git a/docker-compose.yml b/docker-compose.yml index 7e074b0..b60bc33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: ckan: build: diff --git a/docs/_scripts/gen_api_pages.py b/docs/_scripts/gen_api_pages.py index 2a8869d..9a9801d 100644 --- a/docs/_scripts/gen_api_pages.py +++ b/docs/_scripts/gen_api_pages.py @@ -1,6 +1,5 @@ # !/usr/bin/env python # encoding: utf-8 - """ Generate the code reference pages and navigation. diff --git a/pyproject.toml b/pyproject.toml index 0df6da5..2237f27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "tqdm>=4.55.1", "ckanext-ckanpackager>=3.0.0", "ckanext-versioned-datastore>=5.1.0", - "ckantools>=0.3.0" + "ckantools>=0.4.1" ] [project.optional-dependencies] @@ -56,8 +56,8 @@ zip-safe = false exclude = ["tests", "docs"] [tool.setuptools.package-data] -"ckanext.statistics.theme" = ["*", "**/*"] "ckanext.statistics.data" = ["*", "**/*"] +"ckanext.statistics.theme" = ["*", "**/*"] [tool.commitizen] name = "cz_nhm" @@ -70,9 +70,8 @@ version_files = [ "CITATION.cff:^version" ] -[tool.black] -line-length = 88 -skip_string_normalization = true +[tool.ruff.format] +quote-style = "single" [tool.pylint] max-line-length = 88 diff --git a/tests/test_download_statistics.py b/tests/test_download_statistics.py index 7bbd92d..438cc28 100644 --- a/tests/test_download_statistics.py +++ b/tests/test_download_statistics.py @@ -7,18 +7,18 @@ from collections import OrderedDict from datetime import datetime +from unittest.mock import MagicMock, call import pytest + from ckanext.ckanpackager.model.stat import CKANPackagerStat, ckanpackager_stats_table from ckanext.statistics.lib.download_statistics import DownloadStatistics, MonthlyStats from ckanext.statistics.model.gbif_download import GBIFDownload, gbif_downloads_table -from ckanext.versioned_datastore.model import stats, slugs, details, downloads +from ckanext.versioned_datastore.model import details, downloads, slugs, stats from ckanext.versioned_datastore.model.downloads import ( - DownloadRequest, CoreFileRecord, - DerivativeFileRecord, + DownloadRequest, ) -from unittest.mock import MagicMock, call class TestMonthlyStats(object): @@ -409,7 +409,6 @@ def test_add_ckanpackager_stats(self): assert call(datetime(2011, 12, 1), 'blarp', 0) in calls def test_add_versioned_datastore_download_stats(self): - core_record_kwargs = { 'query': {}, 'query_version': 'v12.4.9', @@ -423,7 +422,7 @@ def test_add_versioned_datastore_download_stats(self): resource_hash='abcd', modified=datetime(2019, 1, 1), resource_totals={'resource1': 100, 'resource2': 32}, - **core_record_kwargs + **core_record_kwargs, ) core_record_1.save() @@ -433,7 +432,7 @@ def test_add_versioned_datastore_download_stats(self): resource_hash='efgh', modified=datetime(2018, 5, 10), resource_totals={'resource1': 4}, - **core_record_kwargs + **core_record_kwargs, ) core_record_2.save() @@ -443,7 +442,7 @@ def test_add_versioned_datastore_download_stats(self): resource_hash='ijkl', modified=datetime(2018, 4, 14), resource_totals={'resource3': 189}, - **core_record_kwargs + **core_record_kwargs, ) core_record_3.save()