initial commit

.circleci/config.yml

@@ -0,0 +1,87 @@
+version: 2
+defaults:
+  init_environemnt: &init_environment
+    run: |
+      # Database Creation
+      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+      createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
+      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+      createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+
+      # Database Initialization
+      ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan
+      psql --host=ckan-postgres --username=ckan --dbname=${CKAN_DATASTORE_POSTGRES_DB} --command="CREATE extension tablefunc;"
+      ckan -c test-core-circle-ci.ini db init
+      gunzip .test_durations.gz
+  install_deps: &install_deps
+    run: |
+      # OS Dependencies
+      apt update
+      apt install -y postgresql-client
+  run_tests: &run_tests
+    # Tests Backend, split across containers by segments
+    run: |
+      mkdir -p ~/junit/result
+      python -m pytest ${PYTEST_COMMON_OPTIONS} --splits 4 --group $((CIRCLE_NODE_INDEX+1)) --splitting-algorithm least_duration
+
+  ckan_env: &ckan_env
+    environment:
+      CKAN_DATASTORE_POSTGRES_DB: datastore_test
+      CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+      CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+      CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+      CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+      CKAN_POSTGRES_DB: ckan_test
+      CKAN_POSTGRES_USER: ckan_default
+      CKAN_POSTGRES_PWD: pass
+      PGPASSWORD: ckan
+      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=~/junit/result/junit.xml
+  pg_image: &pg_image
+    image: postgres:10
+    environment:
+      POSTGRES_USER: ckan
+      POSTGRES_PASSWORD: ckan
+    name: ckan-postgres
+
+  redis_image: &redis_image
+    image: redis:3
+    name: ckan-redis
+
+  solr_image: &solr_image
+    image: ckan/ckan-solr:master
+    name: ckan-solr
+
+jobs:
+  test-python-3:
+    docker:
+      - image: python:3.7-bullseye
+        <<: *ckan_env
+      - <<: *pg_image
+      - <<: *redis_image
+      - <<: *solr_image
+
+    parallelism: 4
+
+    steps:
+      - checkout
+
+      - <<: *install_deps
+      - run: |
+          # Python Dependencies
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          python setup.py develop
+          pip check
+      - <<: *init_environment
+
+      # Tests Backend, split across containers by segments
+      - <<: *run_tests
+      - store_test_results:
+          path: ~/junit
+      - run: coveralls
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - test-python-3

.devcontainer/Dockerfile

@@ -0,0 +1,23 @@
+# [Choice] Python version: 3, 3.8, 3.7, 3.6
+ARG VARIANT=3
+FROM mcr.microsoft.com/vscode/devcontainers/python:${VARIANT}
+
+ENV PYTHONUNBUFFERED 1
+
+# Update args in docker-compose.yaml to set the UID/GID of the "vscode" user.
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN if [ "$USER_GID" != "1000" ] || [ "$USER_UID" != "1000" ]; then groupmod --gid $USER_GID vscode && usermod --uid $USER_UID --gid $USER_GID vscode; fi
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+     && apt-get -y install --no-install-recommends \
+     postgresql-client

.devcontainer/README_codespaces.md

@@ -0,0 +1,83 @@
+# CKAN in GitHub Codespaces
+
+Welcome to your cloud development instance of CKAN!
+
+⌛ If you are not seeing a browser tab with the CKAN homepage to the right or a terminal below that says _"Running CKAN on http://localhost:5000"_, wait a bit, things are getting set up... ⌛
+
+Once you see them, you are ready to go! 🚀
+
+## What is this?
+
+This is an online development environment powered by [GitHub Codespaces](https://github.com/features/codespaces). It is a fully functional CKAN instance that you can configure and customize in any way you need. Changes that you make to the source files with the editor as well as changes to the site itself (e.g. creating a dataset or uploading a file) will be persisted until you delete the codespace.
+
+⚠️  **Note:** GitHub Codespaces have a free tier, currently 120 core-hours (i.e. 60h on the default 2-core VM, 30h on a 4-core VM), but after that you will be charged for usage. Check the [documentation](https://github.com/features/codespaces) for more details. To check your current usage, go to the [Billing page](https://github.com/settings/billing) in your profile. ⚠️
+
+
+## What can I do with it?
+
+### Explore
+
+It's your own CKAN demo site! You can log in using the `ckan_admin` sysadmin user (password `test1234`) which will give you full control of the UI. Try creating an Organization, adding some datasets, uploading data, etc
+
+* [User Guide](https://docs.ckan.org/en/latest/user-guide.html)
+* [Sysadmin Guide](https://docs.ckan.org/en/latest/sysadmin-guide.html)
+
+
+### Customize
+
+The site has been configured using the default settings that you get in a brand new CKAN instance but you can change any configuration in the `ckan.ini` file. The development server will refresh automatically as soon as you save your changes to reflect the new configuration.
+
+ * [Configuration options reference](https://docs.ckan.org/en/latest/maintaining/configuration.html#ckan-configuration-file)
+ * [Authorization overview](https://docs.ckan.org/en/latest/maintaining/authorization.html)
+
+Additionally, you can install as many extensions as you want. Check the extension README for any particular instructions but they all basically follow the same pattern:
+1. Open a new terminal in the panel below
+2. Clone the extension
+    ```
+    git clone https://github.com/ckan/ckanext-dcat.git
+    ```
+3. Install the extension
+    ```
+    cd ckanext-dcat
+    python setup.py develop --user
+    ```
+3. Install extra requirements (if any)
+    ```
+    pip install -r requirements.txt
+    ```
+4. Add the plugin(s) to the `ckan.plugins` configuration option in the `ckan.ini` file.
+
+### Develop
+
+What you are using right now is an online editor, Visual Studio Code for the Web, which runs in your browser. You can browse the files in the CKAN source code using the tree panel on the left, open one of them and edit it. Once you save your changes, the development server will be restarted automatically.
+
+You can commit your changes to the branch where you started the codespace in using the "Source Control" icon in the left toolbar.
+
+* [Getting started with Visual Studio Code](https://code.visualstudio.com/docs/editor/codebasics)
+* [CKAN Architecture Overview](https://docs.ckan.org/en/latest/contributing/architecture.html)
+
+#### Database
+
+You can run queries against the PostgreSQL database using the "SQLTools" plugin, the database icon in the left toolbar.
+
+* [SQLTools documentation](https://vscode-sqltools.mteixeira.dev/en/home/#features)
+
+#### Tests
+
+To run the automated tests simply add a new terminal to the console below and run the `pytest` command:
+
+```
+pytest --ckan-ini=test-core.ini ckan ckanext
+```
+
+Or to run a specific test:
+
+```
+pytest --ckan-ini=test-core.ini ckan/tests/logic/action/test_create.py::TestMemberCreate::test_group_member_creation
+```
+
+## I need more!
+
+* If you are finding the codespace too slow you can change the machine type to add more cores to the VM, but note that this will count towards your free allowed quota.
+* If you don't like the editor, you can run the same codespace using your local [Visual Studio Code](https://docs.github.com/en/codespaces/developing-in-codespaces/using-github-codespaces-in-visual-studio-code) or [JetBrains IDE](https://docs.github.com/en/codespaces/developing-in-codespaces/using-github-codespaces-in-your-jetbrains-ide) instances.
+* Alternatively you can use a [Docker Compose setup](https://github.com/ckan/ckan-docker) or do a tried and tested [Source Install](https://docs.ckan.org/en/latest/maintaining/installing/install-from-source.html) in your local computer.

.devcontainer/devcontainer.json

@@ -0,0 +1,70 @@
+// Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6
+{
+	"name": "CKAN 2.10",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "ckan",
+	"workspaceFolder": "/workspace",
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {
+		"sqltools.connections": [{
+			"name": "CKAN PostgreSQL DB",
+			"driver": "PostgreSQL",
+			"previewLimit": 50,
+			"server": "localhost",
+			"port": 5432,
+			"database": "ckan_default",
+			"username": "ckan_default",
+			"password": "pass"
+		}],
+		"python.pythonPath": "/usr/local/bin/python",
+		"python.languageServer": "Pylance",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+		"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest"
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance",
+		"mtxr.sqltools",
+		"mtxr.sqltools-driver-pg"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [5000, 5432, 8983, 6379, 8800],
+
+	// Open in browser
+    "portsAttributes": {
+       "5000": {
+         "label": "CKAN",
+         "onAutoForward": "openPreview"
+       }
+     },
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"onCreateCommand": "pip install --user -r requirements.txt && pip install --user -r dev-requirements.txt",
+	"postCreateCommand": "./.devcontainer/setup.sh",
+	"postAttachCommand": "ckan run",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+
+	"customizations": {
+		"codespaces": {
+		  "openFiles": [
+			".devcontainer/README_codespaces.md"
+		  ]
+		}
+	}
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,64 @@
+version: '3'
+
+services:
+  ckan:
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile
+      args:
+        # [Choice] Python version: 3, 3.8, 3.7, 3.6
+        VARIANT: 3.9
+        # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+        NODE_VERSION: "lts/*"
+        # On Linux, you may need to update USER_UID and USER_GID below if not your local UID is not 1000.
+        USER_UID: 1000
+        USER_GID: 1000
+
+    volumes:
+      - ..:/workspace:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity
+
+    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
+
+    # Uncomment the next line to use a non-root user for all processes.
+    # user: vscode
+
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+  db:
+    image: postgres:14
+    restart: unless-stopped
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+      - ./postgres-docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+    environment:
+      POSTGRES_USER: ckan_default
+      POSTGRES_DB: ckan_default
+      POSTGRES_PASSWORD: pass
+    network_mode: service:ckan
+    # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+  solr:
+    container_name: solr
+    image: ckan/ckan-solr:2.9-solr8
+    network_mode: service:ckan
+    volumes:
+      - solr-data:/var/solr
+
+  redis:
+    network_mode: service:ckan
+    container_name: redis
+    image: redis:alpine
+
+  datapusher:
+    container_name: datapusher
+    network_mode: service:ckan
+    image: ckan/ckan-base-datapusher:0.0.19
+
+volumes:
+  postgres-data:
+  solr-data:

.devcontainer/postgres-docker-entrypoint-initdb.d/02_create_datastore_db_and_users.sql

@@ -0,0 +1,2 @@
+CREATE ROLE datastore_default NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD 'pass';
+CREATE DATABASE datastore_default OWNER ckan_default ENCODING 'utf-8';

.devcontainer/postgres-docker-entrypoint-initdb.d/create_test_db.sql

@@ -0,0 +1 @@
+CREATE DATABASE ckan_test OWNER ckan_default ENCODING 'utf-8';

.devcontainer/setup.sh

@@ -0,0 +1,29 @@
+# Install CKAN locally
+python setup.py develop --user
+
+# Create ini file
+ckan generate config ckan.ini
+
+# Set up storage
+mkdir /workspace/data
+ckan config-tool ckan.ini ckan.storage_path=/workspace/data
+
+# Set up site URL
+ckan config-tool ckan.ini ckan.site_url=https://$CODESPACE_NAME-5000.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN
+
+# Init DB
+ckan db init
+
+# Create sysadmin user
+ckan user add ckan_admin [email protected] password=test1234
+ckan sysadmin add ckan_admin
+
+# Set up DataStore + DataPusher
+ckan config-tool ckan.ini "ckan.datapusher.api_token=$(ckan user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')"
+ckan config-tool ckan.ini \
+    ckan.datastore.write_url=postgresql://ckan_default:pass@localhost/datastore_default \
+    ckan.datastore.read_url=postgresql://datastore_default:pass@localhost/datastore_default \
+    ckan.datapusher.url=http://localhost:8800 \
+    ckan.datapusher.callback_url_base=http://localhost:5000 \
+    "ckan.plugins=activity datastore datapusher datatables_view"
+ckan datastore set-permissions | psql $(grep ckan.datastore.write_url ckan.ini | awk -F= '{print $2}')

.editorconfig

@@ -0,0 +1,17 @@
+# EditorConfig http://EditorConfig.org
+
+# Project Root
+root = true
+
+# Default Code Style
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+# Python Code Style
+[*.py]
+indent_size = 4