Ckan 2.10 upgrade (#88)

datopian · Sep 23, 2024 · b90327f · b90327f
1 parent d3bdc17
commit b90327f
Show file tree

Hide file tree

Showing 34 changed files with 1,490 additions and 367 deletions.
diff --git a/.docker-compose-db.yaml b/.docker-compose-db.yaml
@@ -1,50 +1,47 @@
-version: '3.2'
-
 services:
-
   db:
     image: viderum/ckan-cloud-docker:db-latest
     build:
       context: db
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/db-secrets.sh
+      - docker-compose/db-secrets.sh
     volumes:
-    - db:/var/lib/postgresql/data
+      - db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   jobs-db:
     image: postgres
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/db-secrets.sh
+      - docker-compose/db-secrets.sh
     volumes:
-    - jobs-db:/var/lib/postgresql/data
+      - jobs-db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   datastore-db:
     image: viderum/ckan-cloud-docker:db-latest
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/datastore-db-secrets.sh
+      - docker-compose/datastore-db-secrets.sh
     volumes:
-    - datastore-db:/var/lib/postgresql/data
+      - datastore-db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   ckan:
     depends_on:
-    - db
-    - jobs-db
-    - datastore-db
+      - db
+      - jobs-db
+      - datastore-db
 
 volumes:
   db:

diff --git a/.docker-compose.vital-strategies-theme.yaml b/.docker-compose.vital-strategies-theme.yaml
@@ -1,5 +1,3 @@
-version: '3.2'
-
 services:
 
   proxy:
@@ -21,34 +19,35 @@ services:
     build:
       context: ckan
       args:
-        CKAN_BRANCH: ckan-2.7.3
+        CKAN_BRANCH: ckan-2.10.4
         EXTRA_PACKAGES: cron
         EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/"
-        PRE_INSTALL: "sed  -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt"
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b cc6c8e6f19f59e6842d370bf7ac87d94e37a2831 &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview && \
-          install_standard_ckan_extension_github -r okfn/ckanext-sentry && \
-          install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \
-          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b fix-null-content-type && \
+          install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \
+          install_standard_ckan_extension_github -r datopian/ckanext-gtm && \
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l es -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l fr -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l km -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l pt_BR -f && \
-          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_CN -f
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_Hans_CN -f
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-
+    #ports: # Uncomment to expose CKAN on localhost for development
+    #  - 5000:5000
 
   jobs:
     image: viderum/ckan-cloud-docker:ckan-latest-vital-strategies-theme
     build:
       context: ckan
       args:
-        CKAN_BRANCH: ckan-2.7.3
+        CKAN_BRANCH: ckan-2.10.4
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r keitaroinc/ckanext-s3filestore -b main &&\
-          install_standard_ckan_extension_github -r datopian/ckanext-querytool &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b cc6c8e6f19f59e6842d370bf7ac87d94e37a2831 &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-

diff --git a/DEPLOYING.md b/DEPLOYING.md
@@ -115,7 +115,7 @@ In addition to SSL specific configuration, there is one more line you need to ad
 
 This should be enough for the basic installation. In case you need to tweak versions or other initialization parameters for CKAN, you need these two files:
 
-* `docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini`
+* `docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini`
   This is the file used to generate the CKAN main configuration file.
 
 * `.docker-compose.{instance-id}-theme.yaml`
@@ -196,7 +196,7 @@ bash migrate_filestorage.sh $HOST $ACCESS_KEY $SECRET_KEY $BUCKET $STORAGE_PATH
 
 After migration rebuild the SOLR search index.
 ```
-sudo make shell O=<<instance-id>> S=ckan C='/usr/local/bin/ckan-paster --plugin=ckan search-index rebuild  -c /etc/ckan/production.ini'
+sudo make shell O=<<instance-id>> S=ckan C='/usr/local/bin/ckan-paster --plugin=ckan search-index rebuild  -c /etc/ckan/ckan.ini'
 ```
 
 ## Debugging
@@ -255,10 +255,10 @@ POST_INSTALL: |
   install_standard_ckan_extension_github -r datopian/ckanext-s3filestore &&\
 ```
 
-And add extension to the list of plugins in `docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template`
+And add extension to the list of plugins in `docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template`
 
 ```
-# in docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template
+# in docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template
 ckan.plugins = image_view
    ...
    stats
@@ -268,7 +268,7 @@ ckan.plugins = image_view
 Note: depending on extension you might also need to update extensions related configurations in the same file. If needed this type of information is ussually included in extension REAMDE.
 
 ```
-# in docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template
+# in docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template
 ckanext.s3filestore.aws_access_key_id = Your-Access-Key-ID
 ckanext.s3filestore.aws_secret_access_key = Your-Secret-Access-Key
 ckanext.s3filestore.aws_bucket_name = a-bucket-to-store-your-stuff

diff --git a/Makefile b/Makefile
@@ -2,20 +2,29 @@
 
 COMPOSE_FILES = -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml
 
+DATAPUSHER_TYPE ?= datapusher-plus
+CKAN_DB_NAME ?= ckan
+CKAN_DB_USERNAME ?= ckan
+DB_USERNAME ?= postgres
+DATASTORE_DB_NAME ?= datastore
+DATASTORE_DB_USERNAME ?= postgres
+
 start:
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
 
 stop:
 	docker-compose $(COMPOSE_FILES) stop
 
 build:
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) build
 
 pull:
 	docker-compose $(COMPOSE_FILES) pull
 
 shell:
-	docker-compose $(COMPOSE_FILES) exec $S $C
+	docker-compose $(COMPOSE_FILES) exec -it $S sh -c 'if command -v bash > /dev/null 2>&1; then exec bash; else exec sh; fi'
 
 down:
 	docker-compose $(COMPOSE_FILES) down
@@ -36,10 +45,10 @@ exec:
 	docker-compose $(COMPOSE_FILES) exec $S $C
 
 user:
-	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan ckan -c /etc/ckan/ckan.ini user add $U password=$P email=$E
 
 sysadmin:
-	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan ckan -c /etc/ckan/ckan.ini sysadmin add $U
 
 secret:
 	python create_secrets.py
@@ -49,6 +58,20 @@ cron:
 
 clean-rebuild:
 	docker-compose $(COMPOSE_FILES) down -v
-	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f
+	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs -r docker rmi -f
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) build --no-cache
-	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
+	docker-compose $(COMPOSE_FILES) up -d nginx && make cron
+
+backup-db:
+	docker-compose $(COMPOSE_FILES) exec -T db pg_dump -U postgres --format=custom -d ckan > ckan.dump
+	docker-compose ${COMPOSE_FILES} exec -T ckan sh -c "cd /var/lib/ckan && tar -czf /tmp/ckan_data.tar.gz data"
+	docker cp $$(docker-compose ${COMPOSE_FILES} ps -q ckan):/tmp/ckan_data.tar.gz ckan_data.tar.gz
+	docker-compose $(COMPOSE_FILES) exec -T datastore-db pg_dump -U postgres --format=custom -d datastore > datastore.dump
+
+upgrade-db:
+	./db/migration/upgrade_databases.sh "$(COMPOSE_FILES)" "$(CKAN_DB_NAME)" "$(CKAN_DB_USERNAME)" "$(DB_USERNAME)" "$(DATASTORE_DB_NAME)" "$(DATASTORE_DB_USERNAME)"
+
+config-upgrade:
+	./configs_diff.sh