From d9fcc2ba44e608ff1afde3c894e2082fb4a99ef0 Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Tue, 13 Aug 2024 14:41:21 -0400 Subject: [PATCH] [Migration] Allow variable DB names and users --- Makefile | 10 +++++-- README.md | 4 +-- db/migration/upgrade_databases.sh | 13 +++++---- .../vital-strategies-theme-ckan.ini.template | 28 +++++++++---------- docker-compose/ckan-secrets.dat | 2 +- 5 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 4427a89..33ed30e 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,13 @@ COMPOSE_FILES = -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml -DATAPUSHER_TYPE ?= datapusher +DATAPUSHER_TYPE ?= datapusher-plus +CKAN_DB_NAME ?= ckan +CKAN_DB_USER ?= ckan +DB_USERNAME ?= postgres +CKAN_DB_USERNAME ?= ckan +DATASTORE_DB_NAME ?= datastore +DATASTORE_DB_USERNAME ?= postgres start: @export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \ @@ -66,4 +72,4 @@ backup-db: docker-compose $(COMPOSE_FILES) exec -T datastore-db pg_dump -U postgres --format=custom -d datastore > datastore.dump upgrade-db: - ./db/migration/upgrade_databases.sh "$(COMPOSE_FILES)" + ./db/migration/upgrade_databases.sh "$(COMPOSE_FILES)" "$(CKAN_DB_NAME)" "$(CKAN_DB_USERNAME)" "$(DB_USERNAME)" "$(CKAN_DB_USERNAME)" "$(DATASTORE_DB_NAME)" "$(DATASTORE_DB_USERNAME)" diff --git a/README.md b/README.md index 6300007..55c14b1 100644 --- a/README.md +++ b/README.md @@ -417,10 +417,10 @@ All of the following commands should be run in `ckan-cloud-docker` (unless state 2. Backup the DBs: `make backup-db O=vital-strategies` (confirm that you have `ckan.dump`, `datastore.dump` and `ckan_data.tar.gz` in the current directory after running this command) 3. Stop the containers: `make stop O=vital-strategies` 4. Pull the latest changes: `git pull` -5. Specify if you want to use `datapusher-plus` (you can skip this step to stick with vanilla `datapusher`): `export DATAPUSHER_TYPE=datapusher-plus` (always do this in the current terminal session before starting up CKAN 2.10/Python 3 with `ckan-cloud-docker`) +5. (optional and not recommended) If you don't want to use `datapusher-plus`, you will need to export this variable every time you start, stop, or build CKAN: `export DATAPUSHER_TYPE=datapusher` 6. Create secrets: `make secret` (follow the prompts) 7. Clean and rebuild: `make clean-rebuild O=vital-strategies` -8. Run the upgrade script: `make upgrade-db O=vital-strategies` +8. Run the upgrade script: `make upgrade-db O=vital-strategies` (if you have set custom DB names and users, you will need to pass in these options as needed: `make upgrade-db O=vital-strategies CKAN_DB_NAME= DB_USERNAME= CKAN_DB_USERNAME= DATASTORE_DB_NAME= DATASTORE_DB_USERNAME=`— the default values are: `CKAN_DB_NAME=ckan`, `DB_USERNAME=postgres`, `CKAN_DB_USERNAME=ckan`, `DATASTORE_DB_NAME=datastore`, `DATASTORE_DB_USERNAME=postgres`) 9. Stop the containers: `make stop O=vital-strategies` (copy the API token that's output at the end, for the next step) 10. Run `make secret` again and paste the token when prompted (step 13—"Enter Datapusher API token") 11. Start the containers: `make start O=vital-strategies` diff --git a/db/migration/upgrade_databases.sh b/db/migration/upgrade_databases.sh index 2b3928e..16f7312 100755 --- a/db/migration/upgrade_databases.sh +++ b/db/migration/upgrade_databases.sh @@ -1,18 +1,21 @@ #!/bin/bash COMPOSE_FILES=$1 +CKAN_DB_NAME=$2 +DB_USERNAME=$3 +CKAN_DB_USERNAME=$4 +DATASTORE_DB_NAME=$5 +DATSTORE_DB_USERNAME=$6 + CKAN_BACKUP_FILE="ckan.dump" DATASTORE_BACKUP_FILE="datastore.dump" -CKAN_DB_NAME="ckan" -DATASTORE_DB_NAME="datastore" CKAN_DATA_BACKUP_FILE="ckan_data.tar.gz" CKAN_SERVICE="ckan" DB_SERVICE="db" DATASTORE_SERVICE="datastore-db" -DB_USERNAME="postgres" CKAN_CONFIG_PATH="/etc/ckan/ckan.ini" -CKAN_DB_USERNAME="ckan" -DATSTORE_DB_USERNAME="postgres" + + if [ ! -f $CKAN_BACKUP_FILE ]; then echo "" diff --git a/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template b/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template index 1ee370c..6b98ba6 100644 --- a/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template +++ b/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template @@ -133,16 +133,16 @@ ckan.datapusher.url = {{CKAN_DATAPUSHER_URL}} ## AWS S3 settings -# ckanext.cloudstorage.driver = S3_US_EAST2 -# ckanext.cloudstorage.container_name = vital-strategies -# ckanext.cloudstorage.driver_options = {"key": "{{AWS_ACCESS_KEY_ID}}", "secret": "{{AWS_SECRET_ACCESS_KEY}}"} -# ckanext.s3filestore.host_name = https://vital-strategies-ckan.s3.us-east-2.amazonaws.com -# ckanext.s3filestore.aws_storage_path = demo -# ckanext.s3filestore.aws_access_key_id = {{AWS_ACCESS_KEY_ID}} -# ckanext.s3filestore.aws_secret_access_key = {{AWS_SECRET_ACCESS_KEY}} -# ckanext.s3filestore.region_name = us-east-2 -# ckanext.s3filestore.aws_bucket_name = vital-strategies-ckan -# ckanext.s3filestore.signature_version = s3v4 +ckanext.cloudstorage.driver = S3_US_EAST2 +ckanext.cloudstorage.container_name = vital-strategies +ckanext.cloudstorage.driver_options = {"key": "{{AWS_ACCESS_KEY_ID}}", "secret": "{{AWS_SECRET_ACCESS_KEY}}"} +ckanext.s3filestore.host_name = https://vital-strategies-ckan.s3.us-east-2.amazonaws.com +ckanext.s3filestore.aws_storage_path = demo +ckanext.s3filestore.aws_access_key_id = {{AWS_ACCESS_KEY_ID}} +ckanext.s3filestore.aws_secret_access_key = {{AWS_SECRET_ACCESS_KEY}} +ckanext.s3filestore.region_name = us-east-2 +ckanext.s3filestore.aws_bucket_name = vital-strategies-ckan +ckanext.s3filestore.signature_version = s3v4 ## Query Tool Settings @@ -153,10 +153,10 @@ ckanext.querytool.allow_nav_bar = True ## Google Analytics -# googleanalytics.id = {{GA_ID}} -# googleanalytics.account = {{GA_ACCOUNT}} -# googleanalytics.username = {{GA_USERNAME}} -# googleanalytics.password = {{GA_PASSWORD}} +googleanalytics.id = {{GA_ID}} +googleanalytics.account = {{GA_ACCOUNT}} +googleanalytics.username = {{GA_USERNAME}} +googleanalytics.password = {{GA_PASSWORD}} ## Language settings diff --git a/docker-compose/ckan-secrets.dat b/docker-compose/ckan-secrets.dat index 2fef841..27a5f4a 100644 --- a/docker-compose/ckan-secrets.dat +++ b/docker-compose/ckan-secrets.dat @@ -10,7 +10,7 @@ ckan required APP_INSTANCE_UUID 1b05eb54-743a-40a7-8e31-f2c5ff69c0cb Enter Appli ckan required SOLR_URL http://solr:8983/solr/ckan Enter SOLR connection string ckan required CKAN_REDIS_URL redis://redis:6379/1 Enter Redis URL ckan required CKAN_DATAPUSHER_URL http://datapusher:8800/ Enter Datapusher URL -ckan optional CKAN_DATAPUSHER_API_TOKEN empty Enter Datapusher API token +ckan required CKAN_DATAPUSHER_API_TOKEN xxxxxxxx Enter Datapusher API token ckan required SMTP_SERVER mail.example.com Enter SMTP server address ckan required SMTP_USER info Enter SMTP server username ckan optional SMTP_PASSWORD empty Enter SMTP server password