From f414f089b7be5a09aea32c0af21275318340a5f1 Mon Sep 17 00:00:00 2001 From: ctrepka Date: Wed, 22 Mar 2023 15:41:40 -0500 Subject: [PATCH] . --- .github/workflows/action1.yaml | 9 ++++ .gitignore | 2 +- build.sh | 10 ++++ docker/.ckan-env | 41 -------------- docker/.env-ckan.env | 94 +++++++++++++++++++++++++++++++++ docker/.env-database.env | 5 ++ docker/.env-datapusher.env | 26 +++++++++ docker/.env-redis.env | 2 + docker/.env-secrets.env.example | 8 +++ docker/.env-solr8.env | 2 + docker/.env.example | 40 -------------- docker/ckan/Dockerfile | 10 ++-- docker/ckan/setup/app/prerun.py | 2 +- docker/db/postgis | 0 docker/docker-compose.yml | 64 +++++++++------------- scripts/build_image.py | 19 ------- scripts/lib/run.py | 7 --- 17 files changed, 189 insertions(+), 152 deletions(-) create mode 100644 .github/workflows/action1.yaml create mode 100644 build.sh delete mode 100644 docker/.ckan-env create mode 100644 docker/.env-ckan.env create mode 100644 docker/.env-database.env create mode 100644 docker/.env-datapusher.env create mode 100644 docker/.env-redis.env create mode 100644 docker/.env-secrets.env.example create mode 100644 docker/.env-solr8.env delete mode 100644 docker/.env.example delete mode 100644 docker/db/postgis delete mode 100644 scripts/build_image.py delete mode 100644 scripts/lib/run.py diff --git a/.github/workflows/action1.yaml b/.github/workflows/action1.yaml new file mode 100644 index 0000000..47c6858 --- /dev/null +++ b/.github/workflows/action1.yaml @@ -0,0 +1,9 @@ +name: action1 +run-name: ${{ github.actor }} is learning GitHub Actions +on: [push] +jobs: + list-repo: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - run: ls -la \ No newline at end of file diff --git a/.gitignore b/.gitignore index d84fc49..35c435a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,6 @@ docker/.tmp/* **node_modules** -*.env +.env-secrets.env **/*build.log \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..5cae711 --- /dev/null +++ b/build.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +cd docker && \ +docker compose build --build-arg GH_TOKEN=$(aws secretsmanager get-secret-value \ +--secret-id ci-cd \ +--query SecretString \ +--output text | \ +jq .CKAN_GH_CTREPKA_TOKEN | \ +tr -d '"') --progress plain --no-cache 2>&1 | \ +tee build.log \ No newline at end of file diff --git a/docker/.ckan-env b/docker/.ckan-env deleted file mode 100644 index 12afbb8..0000000 --- a/docker/.ckan-env +++ /dev/null @@ -1,41 +0,0 @@ -# Runtime configuration of CKAN enabled through ckanext-envvars -# Information about how it works: https://github.com/okfn/ckanext-envvars -# Note that variables here take presedence over build/up time variables in .env - -# Set to true to disable CKAN from starting and serve a maintenance page -MAINTENANCE_MODE=false - -# General Settings -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN__MAX_UPLOAD_SIZE_MB=512 -CKAN__MAX_RESOURCE_SIZE=512 -# CKAN Plugins -CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher -# CKAN requires storage path to be set in order for filestore to be enabled -CKAN__STORAGE_PATH=/srv/app/data -CKAN__WEBASSETS__PATH=/srv/app/data/webassets -# SYSADMIN settings, a sysadmin user is created automatically with the below credentials -CKAN_SYSADMIN_NAME=sysadmin -CKAN_SYSADMIN_PASSWORD=password -CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com - -# Email settings -CKAN_SMTP_SERVER=smtp.corporateict.domain:25 -CKAN_SMTP_STARTTLS=True -CKAN_SMTP_USER=user -CKAN_SMTP_PASSWORD=pass -CKAN_SMTP_MAIL_FROM=ckan@localhost - -# Datapusher configuration -CKAN__DATAPUSHER__URL=http://datapusher:8000 -CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ - -# Harvest settings -CKAN__HARVEST__MQ__TYPE=redis -CKAN__HARVEST__MQ__HOSTNAME=redis - -# Solr configuration -CKAN_VERSION=2.9.5 -CKAN_CORE_NAME=ckan diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env new file mode 100644 index 0000000..9506c18 --- /dev/null +++ b/docker/.env-ckan.env @@ -0,0 +1,94 @@ +# Runtime configuration of CKAN enabled through ckanext-envvars +# Information about how it works: https://github.com/okfn/ckanext-envvars +# Note that variables here take presedence over build/up time variables in .env + +# Set to true to disable CKAN from starting and serve a maintenance page +MAINTENANCE_MODE=false + + +POSTGRES_PASSWORD=ckan +POSTGRES_USER=postgres +POSTGRES_PORT=5432 + +CKAN_SQLALCHEMY_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/ckan +CKAN_SOLR_URL=http://solr:8983/solr/ckan +CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_SITE_URL=${CKAN_SITE_URL} +CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} + +DATASTORE_READONLY_USER=datastore +DATASTORE_READONLY_PASSWORD=datastore + +#DB CONNECTION STRINGS +CKAN_DATASTORE_WRITE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/datastore +CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore + +# General Settings +CKAN_VERSION=2.9.5 +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 +CKAN__MAX_UPLOAD_SIZE_MB=512 +CKAN__MAX_RESOURCE_SIZE=512 + +# CKAN Plugins +CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher + +# CKAN requires storage path to be set in order for filestore to be enabled +CKAN__STORAGE_PATH=/srv/app/data +CKAN__WEBASSETS__PATH=/srv/app/data/webassets + +# SYSADMIN settings, a sysadmin user is created automatically with the below credentials +CKAN_SYSADMIN_NAME=sysadmin +CKAN_SYSADMIN_PASSWORD=password +CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com + +# Email settings +# CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +# CKAN_SMTP_STARTTLS=True +# CKAN_SMTP_USER=user +# CKAN_SMTP_PASSWORD=pass +# CKAN_SMTP_MAIL_FROM=ckan@localhost + +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ +DATAPUSHER_VERSION=0.0.17 +DATAPUSHER_MAX_CONTENT_LENGTH=512000000 +DATAPUSHER_CHUNK_SIZE=16384 +DATAPUSHER_CHUNK_INSERT_ROWS=250 +DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + +# Harvest settings +CKAN__HARVEST__MQ__TYPE=redis +CKAN__HARVEST__MQ__HOSTNAME=redis + +# Solr configuration +CKAN_VERSION=2.9.5 +CKAN_CORE_NAME=ckan + +# Redis +REDIS_VERSION=6.0.7 + +# ckanext_envvars for plugins +CKAN___BEAKER__SESSION__DATA_SERIALIZER=json +CKAN___BEAKER__SESSION__HTTPONLY=true +CKAN___BEAKER__SESSION__SECURE=true +CKAN___BEAKER__SESSION__TIMEOUT=3600 +CKAN___BEAKER__SESSION__SAVE_ACCESSED_TIME=true +CKAN___BEAKER__SESSION__TYPE=ext:redis +CKAN___BEAKER__SESSION__URL=redis://redis:6379/8 +CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true +CKAN___BEAKER__SESSION__COOKIE_DOMAIN=localhost:5000 +CKANEXT__SECURITY__DOMAIN=localhost:5000 +CKANEXT__SECURITY__REDIS__HOST=redis +CKANEXT__SECURITY__REDIS__PORT=6379 +CKANEXT__SECURITY__REDIS__DB=8 +CKANEXT__SECURITY__LOCK_TIMEOUT=900 +CKANEXT__SECURITY__LOGIN_MAX_COUNT=10 +CKANEXT__SECURITY__BRUTE_FORCE_KEY=user_name +CKANEXT__SECURITY__DISABLE_PASSWORD_RESET_OVERRIDE=true +CKANEXT__SECURITY__MFA_HELP_LINK=https://localhost:5000/pages/mfa diff --git a/docker/.env-database.env b/docker/.env-database.env new file mode 100644 index 0000000..21c93b4 --- /dev/null +++ b/docker/.env-database.env @@ -0,0 +1,5 @@ +# Database +POSTGRES_PASSWORD=ckan +POSTGRES_USER=postgres +POSTGRES_PORT="5432" +DATASTORE_READONLY_PASSWORD=datastore diff --git a/docker/.env-datapusher.env b/docker/.env-datapusher.env new file mode 100644 index 0000000..3b6003c --- /dev/null +++ b/docker/.env-datapusher.env @@ -0,0 +1,26 @@ +# Datapusher +DATAPUSHER_VERSION=0.0.17 +DATAPUSHER_MAX_CONTENT_LENGTH=512000000 +DATAPUSHER_CHUNK_SIZE=16384 +DATAPUSHER_CHUNK_INSERT_ROWS=250 +DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5000 + +DB_HOST=db +POSTGRES_USER=postgres +POSTGRES_PASSWORD=ckan +POSTGRES_DB=ckan +CKAN_DB_USER=ckan +CKAN_DB_PASSWD=ckan +CKAN_DB_NAME=ckan +DATASTORE_DB_USER=datastore +DATASTORE_DB_PASSWORD=datastore +DATASTORE_DB_NAME=datastore +JOBS_DB_USER=jobs +JOBS_DB_PASSWORD=jobs +JOBS_DB_NAME=jobs +CKAN_SQLALCHEMY_URL=postgresql://$(CKAN_DB_USER):$(CKAN_DB_PASSWD)@$(DB_HOST)/$(CKAN_DB_NAME) +SQLALCHEMY_DATABASE_URI=postgresql://$(JOBS_DB_USER):$(JOBS_DB_PASSWORD)@$(DB_HOST)/$(JOBS_DB_NAME) +WRITE_ENGINE_URL=postgresql://$(DATASTORE_DB_USER):$(DATASTORE_DB_PASSWORD)@$(DB_HOST)/$(DATASTORE_DB_NAME) \ No newline at end of file diff --git a/docker/.env-redis.env b/docker/.env-redis.env new file mode 100644 index 0000000..87f22d4 --- /dev/null +++ b/docker/.env-redis.env @@ -0,0 +1,2 @@ +# Redis +REDIS_VERSION=6.0.7 \ No newline at end of file diff --git a/docker/.env-secrets.env.example b/docker/.env-secrets.env.example new file mode 100644 index 0000000..229e1a8 --- /dev/null +++ b/docker/.env-secrets.env.example @@ -0,0 +1,8 @@ +# DO NOT UPLOAD THESE VALUES TO GITHUB!!! +# THIS FILE SHOULD BE RENAMED TO `.env-secrets.env` WHEN TESTING LOCALLY +# AGAIN, DO NOT UPLOAD THESE VALUES TO SOURCE CONTROL +# THEY ALLOW ACCESS TO A PRIVATE S3 BUCKET + +# s3filestore key id and access key +CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID= +CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY= \ No newline at end of file diff --git a/docker/.env-solr8.env b/docker/.env-solr8.env new file mode 100644 index 0000000..d2fff38 --- /dev/null +++ b/docker/.env-solr8.env @@ -0,0 +1,2 @@ +# SOLR +CKAN_CORE_NAME=ckan diff --git a/docker/.env.example b/docker/.env.example deleted file mode 100644 index e0f19c6..0000000 --- a/docker/.env.example +++ /dev/null @@ -1,40 +0,0 @@ -# Variables in this file will be used as build arguments when running -# docker-compose build and docker-compose up -# Verify correct substitution with "docker-compose config" -# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: -# docker-compose down -v -# docker-compose build -# docker-compose up -d - -# Database -POSTGRES_PASSWORD=ckan -POSTGRES_USER=ckan -POSTGRES_PORT=5432 -DATASTORE_READONLY_PASSWORD=datastore - -# CKAN -CKAN_VERSION=2.9.5 -CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5000 -CKAN_PORT=5000 -CKAN_MAX_UPLOAD_SIZE_MB=512 - -# Datapusher -DATAPUSHER_VERSION=0.0.17 -DATAPUSHER_MAX_CONTENT_LENGTH=512000000 -DATAPUSHER_CHUNK_SIZE=16384 -DATAPUSHER_CHUNK_INSERT_ROWS=250 -DATAPUSHER_DOWNLOAD_TIMEOUT=30 -DATAPUSHER_SSL_VERIFY=False -DATAPUSHER_REWRITE_RESOURCES=True -DATAPUSHER_REWRITE_URL=http://ckan:5000 - -# SOLR -CKAN_CORE_NAME=ckan - -# Redis -REDIS_VERSION=6.0.7 - -# S3 FILESTORE ENVVARS -CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID= -CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY= diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile index 7d32119..beb9599 100644 --- a/docker/ckan/Dockerfile +++ b/docker/ckan/Dockerfile @@ -228,10 +228,12 @@ RUN pip install --no-index --find-links=/srv/app/wheels -r requirements.txt RUN pip install future # Generate CKAN config -RUN ckan generate config ${APP_DIR}/production.ini && \ - python ${SRC_DIR}/plugins/plugins.py && \ - # Configure plugins - ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ +RUN ckan generate config ${APP_DIR}/production.ini +# Install plugins +RUN python ${SRC_DIR}/plugins/plugins.py + +# Configure plugins +RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ # Create the data directory mkdir ${DATA_DIR} && \ # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created diff --git a/docker/ckan/setup/app/prerun.py b/docker/ckan/setup/app/prerun.py index 3560ba1..f07ba1f 100644 --- a/docker/ckan/setup/app/prerun.py +++ b/docker/ckan/setup/app/prerun.py @@ -213,7 +213,7 @@ def create_sysadmin(): print('[prerun] Maintenance mode, skipping setup...') else: check_db_connection() - #check_solr_connection() + check_solr_connection() init_db() if os.environ.get('CKAN_DATASTORE_WRITE_URL'): init_datastore() diff --git a/docker/db/postgis b/docker/db/postgis deleted file mode 100644 index e69de29..0000000 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5ec3bb1..4b1bc8a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -8,7 +8,10 @@ volumes: services: ckan: container_name: ckan - image: ckan + hostname: ckan + build: + context: ckan + dockerfile: Dockerfile networks: - frontend - backend @@ -16,79 +19,62 @@ services: - db - solr - redis - ports: - - "0.0.0.0:${CKAN_PORT}:5000" env_file: - - ./.ckan-env - - ./.env - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan - - CKAN_SOLR_URL=http://solr:8983/solr/ckan - - CKAN_REDIS_URL=redis://redis:6379/1 - - CKAN_SITE_URL=${CKAN_SITE_URL} - - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} - - DS_RO_PASSWORD=datastore - - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore - - CKAN_DATASTORE_READ_URL=postgresql://datastore:${DATASTORE_READONLY_PASSWORD}@db/datastore + - ./.env-ckan.env + - ./.env-secrets.env + ports: + - "0.0.0.0:5000:5000" volumes: - ckan_data:/srv/app/data datapusher: container_name: datapusher + hostname: datapusher image: 746466009731.dkr.ecr.us-east-1.amazonaws.com/datapusher-plus:0.7.0 + depends_on: + - db + - solr networks: - frontend - backend - ports: - - "8000:8000" env_file: - - ./.ckan-env - - ./.env - environment: - - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} - - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} - - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} - - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} - - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY} - - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} - - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - ./.env-ckan.env + - ./.env-datapusher.env db: container_name: db + hostname: db build: ./db networks: - backend + env_file: + - ./.env-database.env environment: - POSTGRES_HOST_AUTH_METHOD=trust - ports: - - 5432:5432 healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] solr: container_name: solr + hostname: solr image: solr:8.11.1 networks: - backend - ports: - - 8983:8983 env_file: - - ./.ckan-env - environment: - - CKAN_CORE_NAME=${CKAN_CORE_NAME} - - CKAN_VERSION=${CKAN_VERSION} + - ./.env-ckan.env + - ./.env-solr8.env volumes: - solr_data:/var/solr - ./solr8/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + redis: container_name: redis - image: redis:${REDIS_VERSION} + hostname: redis + env_file: + - ./.env-redis.env + image: redis:6.0.7 networks: - backend - ports: - - 6379:6379 networks: frontend: backend: diff --git a/scripts/build_image.py b/scripts/build_image.py deleted file mode 100644 index d5a573f..0000000 --- a/scripts/build_image.py +++ /dev/null @@ -1,19 +0,0 @@ -from lib.run import run -import json -import subprocess - -GITHUB_SECRET_NAME = "ci-cd" - -SECRETS = run( -"aws secretsmanager get-secret-value \ ---secret-id {v} --query SecretString \ ---output text".format( - v=GITHUB_SECRET_NAME, -)) - -TOKEN = json.loads(SECRETS)["CKAN_GH_CTREPKA_TOKEN"] - -DOCKER_OUTPUT = subprocess.call( - f"ls && cd ../ && ls", - shell=True -) \ No newline at end of file diff --git a/scripts/lib/run.py b/scripts/lib/run.py deleted file mode 100644 index de8fb15..0000000 --- a/scripts/lib/run.py +++ /dev/null @@ -1,7 +0,0 @@ -import subprocess - - -def run(cmd: str): - # print(cmd) - result = subprocess.getoutput(cmd) - return result