Fix requirements; Add dev documentation (#87)

.docker-compose.vital-strategies-theme.yaml

@@ -26,7 +26,7 @@ services:
         EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/"
         PRE_INSTALL: "sed  -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt"
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.0.1 &&\
+          install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview && \
           install_standard_ckan_extension_github -r okfn/ckanext-sentry && \
           install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \

Makefile

@@ -1,35 +1,54 @@
-.PHONY: start stop build pull shell down remove logs user sysadmin secret cron
+.PHONY: start stop build pull shell down remove remove-images logs logs-less exec user sysadmin secret cron clean-rebuild
+
+COMPOSE_FILES = -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml
 
 start:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml up -d --build nginx && make cron
+	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
 
 stop:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml stop
+	docker-compose $(COMPOSE_FILES) stop
 
 build:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml build
+	docker-compose $(COMPOSE_FILES) build
 
 pull:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml pull
+	docker-compose $(COMPOSE_FILES) pull
 
 shell:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml exec $S $C
+	docker-compose $(COMPOSE_FILES) exec $S $C
 
 down:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml down
+	docker-compose $(COMPOSE_FILES) down
 
 remove:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml down -v
+	docker-compose $(COMPOSE_FILES) down -v
+
+remove-images:
+	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f
 
 logs:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml logs -f $S
+	docker-compose $(COMPOSE_FILES) logs -f $S
+
+logs-less:
+	docker-compose $(COMPOSE_FILES) logs $S | less
+
+exec:
+	docker-compose $(COMPOSE_FILES) exec $S $C
+
 user:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml \
-	 exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini
+
 sysadmin:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml \
-	 exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini
+
 secret:
 	python create_secrets.py
+
 cron:
-	docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml exec --user=root ckan service cron start
+	docker-compose $(COMPOSE_FILES) exec --user=root ckan service cron start
+
+clean-rebuild:
+	docker-compose $(COMPOSE_FILES) down -v
+	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f
+	docker-compose $(COMPOSE_FILES) build --no-cache
+	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron

README.md

@@ -25,11 +25,99 @@ Install Docker for [Windows](https://store.docker.com/editions/community/docker-
 
 
 ## Generate or update files with secrets
+
 Run and follow all steps:
 ```
 ./create_secrets.py
 ```
 
+If you're setting up a local environment for development and testing, you can leave all of the secret values as is. Just press enter when prompted for a value.
+
+## Running locally for development and testing
+
+If you want to run this locally and don't want to deploy it anywhere, you must make a few changes before you can start the environment. Once you've gone through the steps below and started the environment, you can access CKAN at http://ckan:5000 (when creating secrets, don't change the default site URL).
+
+**Note**: The "Install" and "Generate or update files with secrets" sections above are still required before proceeding with a local setup. After making the changes below, you can continue with the "Running a CKAN instance using the docker-compose environment" section.
+
+### Use traefik.dev.toml
+
+By default, `traefik` will attempt to generate a certificate and use https. This will cause issues with the local development environment. To fix this, you can use the `traefik.dev.toml` by updating the `proxy` service in `docker-compose.yaml` to use the `traefik.dev.toml` file. The dev version doesn't generate a certificate and uses http instead of https.
+
+```
+  proxy:
+    image: traefik:1.7.2-alpine
+    restart: always
+    volumes:
+      - ./traefik/traefik.dev.toml:/traefik.toml # <-- Replace ./traefik/traefik.toml with ./traefik/traefik.dev.toml as shown here
+      - ./traefik/acme.json:/acme.json
+    networks:
+    - ckan-multi
+```
+
+### Expose port 5000 for CKAN
+
+In your project specific `docker-compose` file, you must expose port 5000 for CKAN. Otherwise, CKAN will not be accessible from the host machine. For example, if you want to run `.docker-compose.vital-strategies-theme.yaml` locally, you would add the ports section as shown below:
+
+```
+  ckan:
+    depends_on:
+      - datapusher
+    links:
+      - datapusher
+    image: viderum/ckan-cloud-docker:ckan-latest-vital-strategies-theme
+    build:
+      context: ckan
+      args:
+        CKAN_BRANCH: ckan-2.7.3
+        EXTRA_PACKAGES: cron
+        EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/"
+        PRE_INSTALL: "sed  -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt"
+        POST_INSTALL: |
+          install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\
+          install_standard_ckan_extension_github -r ckan/ckanext-geoview && \
+          install_standard_ckan_extension_github -r okfn/ckanext-sentry && \
+          install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b fix-null-content-type && \
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l es -f && \
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l fr -f && \
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l km -f && \
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l pt_BR -f && \
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_CN -f
+    environment:
+      - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-
+    ports: # <-- Add this section to expose port 5000
+    - 5000:5000
+```
+
+### Remove unused plugins from CKAN
+
+Before building and starting the environment, make sure you only have the required plugins enabled. If you're using a pre-defined project template for local testing, you might not need some of the included extensions, such as `ckanext-googleanalytics` or `ckanext-sentry`. For example, if you want to use the `vital-strategies` project template, you should remove the following plugins from the `.ini` file (found in `docker-compose/ckan-conf-templates/vital-strategies-theme-production.ini`) to avoid issues (unless you want to properly configure them):
+
+```
+ckan.plugins = image_view
+   text_view
+   recline_view
+   datastore
+   datapusher
+   resource_proxy
+   geojson_view
+   querytool
+   stats
+   sentry # <-- Remove this line
+   s3filestore # <-- Remove this line
+   googleanalytics # <-- Remove this line
+```
+
+### Hosts file entries
+
+When using this environment locally, you must add the following entries to your hosts file (`nginx` is mentioned in the next section, but `ckan` is specific to the development and testing setup):
+
+```
+127.0.0.1  nginx
+127.0.0.1  ckan
+```
+
 ## Running a CKAN instance using the docker-compose environment
 
 (optional) Clear any existing compose environment to ensure a fresh start
@@ -77,6 +165,57 @@ To start the jobs server for uploading to the datastore DB:
 docker-compose up -d jobs
 ```
 
+### Optionally, use make commands
+
+The following commands use the `vital-strategies` project template as an example. Replace `vital-strategies` with the name of your project template. **Note**: Using the commands below still requires adding `nginx` to your hosts file as shown above.
+
+Build the images:
+
+```
+make build O=vital-strategies
+```
+
+Start the environment (this will also build the images if they haven't been built yet):
+
+```
+make start O=vital-strategies
+```
+
+Stop the environment:
+
+```
+make stop O=vital-strategies
+```
+
+Make a user:
+
+```
+make user O=vital-strategies U=USER_NAME P=PASSWORD E=EMAIL
+```
+
+Make a user a sysadmin:
+
+```
+make sysadmin O=vital-strategies U=USER_NAME
+```
+
+Remove the containers and volumes:
+
+```
+make remove O=vital-strategies
+```
+
+Remove the associated images:
+
+```
+make remove-images O=vital-strategies
+```
+
+Completely remove and then rebuild the environment (this will remove containers, volumes, and images):
+
+```
+make clean-rebuild O=vital-strategies
+```
 
 ## Making modifications to the docker images / configuration
 

ckan/Dockerfile

@@ -69,6 +69,8 @@ RUN CKAN_BRANCH="${CKAN_BRANCH:-ckan-2.8.1}" && CKAN_REPO="${CKAN_REPO:-ckan/cka
 ARG PRE_INSTALL
 RUN eval "${PRE_INSTALL}"
 
+RUN sed -i 's/psycopg2==2.4.5/psycopg2==2.7.3.2/g' $CKAN_VENV/src/ckan/requirements.txt
+
 RUN touch $CKAN_VENV/src/ckan/requirement-setuptools.txt && ckan-pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt
 RUN touch $CKAN_VENV/src/ckan/requirements.txt && ckan-pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt
 

docker-compose/ckan-secrets.dat

@@ -14,7 +14,7 @@ ckan required SMTP_SERVER mail.example.com Enter SMTP server address
 ckan required SMTP_USER info Enter SMTP server username
 ckan optional SMTP_PASSWORD empty Enter SMTP server password
 ckan optional SMTP_MAIL_FROM empty Enter SMTP mail from
-ckan optional CKAN_SITE_URL http://ckan.localhost:5000 Enter Website URL (including https://)
+ckan required CKAN_SITE_URL http://ckan:5000 Enter Website URL (including https:// or http://)
 ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key (if any)
 ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key (if any)
 ckan optional SENTRY_DSN https://<token>@sentry.io/<id> Enter Sentry DSN URL with token and ID

jenkins/Dockerfile

@@ -3,9 +3,9 @@ FROM jenkins/jenkins:lts
 ARG PIP_INDEX_URL
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
-RUN /usr/local/bin/install-plugins.sh \
-        build-timeout envfile copyartifact extensible-choice-parameter fail-the-build file-operations \
-        filesystem-list-parameter fstrigger generic-webhook-trigger git-parameter github-branch-source \
+RUN jenkins-plugin-cli --plugins \
+        build-timeout envfile copyartifact extensible-choice-parameter file-operations \
+        fstrigger generic-webhook-trigger git-parameter github-branch-source \
         global-variable-string-parameter http_request jobgenerator join managed-scripts matrix-combinations-parameter \
         persistent-parameter workflow-aggregator pipeline-github-lib python ssh-slaves timestamper urltrigger \
         ws-cleanup
@@ -20,8 +20,6 @@ RUN chmod +x /usr/local/bin/docker-compose && echo "jenkins ALL=NOPASSWD: ALL" >
     echo "export CKAN_CLOUD_DOCKER_JENKINS=1" > /etc/profile.d/ckan_cloud_docker_jenkins &&\
     chmod +x /etc/profile.d/ckan_cloud_docker_jenkins
 RUN apt update && apt install -y python3-pip
-RUN python3 -m pip install --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} pyyaml
+RUN python3 -m pip install --break-system-packages --index-url ${PIP_INDEX_URL:-https://pypi.org/simple/} pyyaml
 
 USER jenkins
-
-RUN /usr/local/bin/install-plugins.sh rebuild

traefik/traefik.dev.toml

@@ -0,0 +1,32 @@
+debug = false
+defaultEntryPoints = ["http"]
+
+[entryPoints]
+    [entryPoints.http]
+      address = ":80"
+
+    [entryPoints.api]
+      address = ":8081"
+
+    [api]
+      entryPoint = "api"
+
+    [ping]
+      entryPoint = "http"
+
+    [accessLog]
+
+    [file]
+      watch = true
+
+    [backends]
+      [backends.ckan]
+        [backends.ckan.servers.server1]
+          url = "http://nginx:8080"
+
+    [frontends]
+      [frontends.ckan]
+        backend="ckan"
+        passHostHeader = true
+        [frontends.ckan.routes.route1]
+          rule = "Host:localhost"