.

.github/workflows/action1.yaml

@@ -0,0 +1,9 @@
+name: action1
+run-name: ${{ github.actor }} is learning GitHub Actions
+on: [push]
+jobs:
+  list-repo:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - run: ls -la

.gitignore

@@ -7,6 +7,6 @@ docker/.tmp/*
 
 **node_modules**
 
-*.env
+.env-secrets.env
 
 **/*build.log

build.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+cd docker && \
+docker compose build --build-arg GH_TOKEN=$(aws secretsmanager get-secret-value \
+--secret-id ci-cd \
+--query SecretString \
+--output text | \
+jq .CKAN_GH_CTREPKA_TOKEN | \
+tr -d '"') --progress plain --no-cache 2>&1 | \
+tee build.log

docker/.env-ckan.env

@@ -0,0 +1,94 @@
+# Runtime configuration of CKAN enabled through ckanext-envvars
+# Information about how it works: https://github.com/okfn/ckanext-envvars
+# Note that variables here take presedence over build/up time variables in .env
+
+# Set to true to disable CKAN from starting and serve a maintenance page
+MAINTENANCE_MODE=false
+
+
+POSTGRES_PASSWORD=ckan
+POSTGRES_USER=postgres
+POSTGRES_PORT=5432
+
+CKAN_SQLALCHEMY_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/ckan
+CKAN_SOLR_URL=http://solr:8983/solr/ckan
+CKAN_REDIS_URL=redis://redis:6379/1
+CKAN_SITE_URL=${CKAN_SITE_URL}
+CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
+
+DATASTORE_READONLY_USER=datastore
+DATASTORE_READONLY_PASSWORD=datastore
+
+#DB CONNECTION STRINGS
+CKAN_DATASTORE_WRITE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/datastore
+CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
+
+# General Settings
+CKAN_VERSION=2.9.5
+CKAN_SITE_ID=default
+CKAN_SITE_URL=http://localhost:5000
+CKAN_PORT=5000
+CKAN__MAX_UPLOAD_SIZE_MB=512
+CKAN__MAX_RESOURCE_SIZE=512
+
+# CKAN Plugins
+CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher
+
+# CKAN requires storage path to be set in order for filestore to be enabled
+CKAN__STORAGE_PATH=/srv/app/data
+CKAN__WEBASSETS__PATH=/srv/app/data/webassets
+
+# SYSADMIN settings, a sysadmin user is created automatically with the below credentials
+CKAN_SYSADMIN_NAME=sysadmin
+CKAN_SYSADMIN_PASSWORD=password
+CKAN_SYSADMIN_EMAIL=[email protected]
+
+# Email settings
+# CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+# CKAN_SMTP_STARTTLS=True
+# CKAN_SMTP_USER=user
+# CKAN_SMTP_PASSWORD=pass
+# CKAN_SMTP_MAIL_FROM=ckan@localhost
+
+# Datapusher configuration
+CKAN__DATAPUSHER__URL=http://datapusher:8000
+CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/
+DATAPUSHER_VERSION=0.0.17
+DATAPUSHER_MAX_CONTENT_LENGTH=512000000
+DATAPUSHER_CHUNK_SIZE=16384
+DATAPUSHER_CHUNK_INSERT_ROWS=250
+DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
+
+# Harvest settings
+CKAN__HARVEST__MQ__TYPE=redis
+CKAN__HARVEST__MQ__HOSTNAME=redis
+
+# Solr configuration
+CKAN_VERSION=2.9.5
+CKAN_CORE_NAME=ckan
+
+# Redis
+REDIS_VERSION=6.0.7
+
+# ckanext_envvars for plugins
+CKAN___BEAKER__SESSION__DATA_SERIALIZER=json
+CKAN___BEAKER__SESSION__HTTPONLY=true
+CKAN___BEAKER__SESSION__SECURE=true
+CKAN___BEAKER__SESSION__TIMEOUT=3600
+CKAN___BEAKER__SESSION__SAVE_ACCESSED_TIME=true
+CKAN___BEAKER__SESSION__TYPE=ext:redis
+CKAN___BEAKER__SESSION__URL=redis://redis:6379/8
+CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true
+CKAN___BEAKER__SESSION__COOKIE_DOMAIN=localhost:5000
+CKANEXT__SECURITY__DOMAIN=localhost:5000
+CKANEXT__SECURITY__REDIS__HOST=redis
+CKANEXT__SECURITY__REDIS__PORT=6379
+CKANEXT__SECURITY__REDIS__DB=8
+CKANEXT__SECURITY__LOCK_TIMEOUT=900
+CKANEXT__SECURITY__LOGIN_MAX_COUNT=10
+CKANEXT__SECURITY__BRUTE_FORCE_KEY=user_name
+CKANEXT__SECURITY__DISABLE_PASSWORD_RESET_OVERRIDE=true
+CKANEXT__SECURITY__MFA_HELP_LINK=https://localhost:5000/pages/mfa

docker/.env-database.env

@@ -0,0 +1,5 @@
+# Database
+POSTGRES_PASSWORD=ckan
+POSTGRES_USER=postgres
+POSTGRES_PORT="5432"
+DATASTORE_READONLY_PASSWORD=datastore

docker/.env-datapusher.env

@@ -0,0 +1,26 @@
+# Datapusher
+DATAPUSHER_VERSION=0.0.17
+DATAPUSHER_MAX_CONTENT_LENGTH=512000000
+DATAPUSHER_CHUNK_SIZE=16384
+DATAPUSHER_CHUNK_INSERT_ROWS=250
+DATAPUSHER_DOWNLOAD_TIMEOUT=30
+DATAPUSHER_SSL_VERIFY=False
+DATAPUSHER_REWRITE_RESOURCES=True
+DATAPUSHER_REWRITE_URL=http://ckan:5000
+
+DB_HOST=db
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=ckan
+POSTGRES_DB=ckan
+CKAN_DB_USER=ckan
+CKAN_DB_PASSWD=ckan
+CKAN_DB_NAME=ckan
+DATASTORE_DB_USER=datastore
+DATASTORE_DB_PASSWORD=datastore
+DATASTORE_DB_NAME=datastore
+JOBS_DB_USER=jobs
+JOBS_DB_PASSWORD=jobs
+JOBS_DB_NAME=jobs
+CKAN_SQLALCHEMY_URL=postgresql://$(CKAN_DB_USER):$(CKAN_DB_PASSWD)@$(DB_HOST)/$(CKAN_DB_NAME)
+SQLALCHEMY_DATABASE_URI=postgresql://$(JOBS_DB_USER):$(JOBS_DB_PASSWORD)@$(DB_HOST)/$(JOBS_DB_NAME)
+WRITE_ENGINE_URL=postgresql://$(DATASTORE_DB_USER):$(DATASTORE_DB_PASSWORD)@$(DB_HOST)/$(DATASTORE_DB_NAME)

docker/.env-redis.env

@@ -0,0 +1,2 @@
+# Redis
+REDIS_VERSION=6.0.7

docker/.env-secrets.env.example

@@ -0,0 +1,8 @@
+# DO NOT UPLOAD THESE VALUES TO GITHUB!!!
+# THIS FILE SHOULD BE RENAMED TO `.env-secrets.env` WHEN TESTING LOCALLY
+# AGAIN, DO NOT UPLOAD THESE VALUES TO SOURCE CONTROL
+# THEY ALLOW ACCESS TO A PRIVATE S3 BUCKET
+
+# s3filestore key id and access key
+CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=<ACCESS_KEY_ID_HERE>
+CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=<SECRET_ACCESS_KEY_HERE>

docker/.env-solr8.env

@@ -0,0 +1,2 @@
+# SOLR
+CKAN_CORE_NAME=ckan

docker/ckan/Dockerfile

@@ -228,10 +228,12 @@ RUN pip install --no-index --find-links=/srv/app/wheels -r requirements.txt
 RUN pip install future
 
 # Generate CKAN config
-RUN ckan generate config ${APP_DIR}/production.ini && \
-    python ${SRC_DIR}/plugins/plugins.py && \ 
-    # Configure plugins
-    ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
+RUN ckan generate config ${APP_DIR}/production.ini
+# Install plugins
+RUN python ${SRC_DIR}/plugins/plugins.py
+
+# Configure plugins
+RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
     # Create the data directory
     mkdir ${DATA_DIR} && \
     # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created

docker/ckan/setup/app/prerun.py

@@ -213,7 +213,7 @@ def create_sysadmin():
         print('[prerun] Maintenance mode, skipping setup...')
     else:
         check_db_connection()
-        #check_solr_connection()
+        check_solr_connection()
         init_db()
         if os.environ.get('CKAN_DATASTORE_WRITE_URL'):
             init_datastore()

docker/docker-compose.yml

@@ -8,87 +8,73 @@ volumes:
 services:
   ckan:
     container_name: ckan
-    image: ckan
+    hostname: ckan
+    build:
+      context: ckan
+      dockerfile: Dockerfile
     networks:
       - frontend
       - backend
     depends_on:
       - db
       - solr
       - redis
-    ports:
-      - "0.0.0.0:${CKAN_PORT}:5000"
     env_file:
-      - ./.ckan-env
-      - ./.env
-    environment:
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
-      - CKAN_SOLR_URL=http://solr:8983/solr/ckan
-      - CKAN_REDIS_URL=redis://redis:6379/1
-      - CKAN_SITE_URL=${CKAN_SITE_URL}
-      - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
-      - DS_RO_PASSWORD=datastore
-      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore
-      - CKAN_DATASTORE_READ_URL=postgresql://datastore:${DATASTORE_READONLY_PASSWORD}@db/datastore
+      - ./.env-ckan.env
+      - ./.env-secrets.env
+    ports:
+      - "0.0.0.0:5000:5000"
     volumes:
       - ckan_data:/srv/app/data
 
   datapusher:
     container_name: datapusher
+    hostname: datapusher
     image: 746466009731.dkr.ecr.us-east-1.amazonaws.com/datapusher-plus:0.7.0
+    depends_on:
+      - db
+      - solr
     networks:
       - frontend
       - backend
-    ports:
-      - "8000:8000"
     env_file:
-      - ./.ckan-env
-      - ./.env
-    environment:
-      - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH}
-      - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE}
-      - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS}
-      - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT}
-      - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY}
-      - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES}
-      - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - ./.env-ckan.env
+      - ./.env-datapusher.env
 
   db:
     container_name: db
+    hostname: db
     build: ./db
     networks:
       - backend
+    env_file:
+      - ./.env-database.env
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
-    ports:
-      - 5432:5432
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "postgres"]
 
   solr:
     container_name: solr
+    hostname: solr
     image: solr:8.11.1
     networks:
       - backend
-    ports:
-      - 8983:8983
     env_file:
-      - ./.ckan-env
-    environment:
-      - CKAN_CORE_NAME=${CKAN_CORE_NAME}
-      - CKAN_VERSION=${CKAN_VERSION}
+      - ./.env-ckan.env
+      - ./.env-solr8.env
     volumes:
       - solr_data:/var/solr
       - ./solr8/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+
   redis:
     container_name: redis
-    image: redis:${REDIS_VERSION}
+    hostname: redis
+    env_file:
+      - ./.env-redis.env
+    image: redis:6.0.7
     networks:
       - backend
-    ports:
-      - 6379:6379
 networks:
   frontend:
   backend: