environment documentation and docker-compose updates

VEuPathDB · Apr 9, 2024 · b03ca0f · b03ca0f
1 parent 0c75b00
commit b03ca0f
Show file tree

Hide file tree

Showing 5 changed files with 174 additions and 73 deletions.
diff --git a/components/env/src/main/kotlin/vdi/component/env/EnvKey.kt b/components/env/src/main/kotlin/vdi/component/env/EnvKey.kt
@@ -651,12 +651,12 @@ object EnvKey {
 
   object Reconciler {
     /**
-     * Whether the reconciler should be enabled at all.
+     * Whether the full dataset reconciliation daemon should be enabled at all.
      *
      * Type: Boolean
      * Required: no
      */
-    const val Enabled = "RECONCILER_ENABLED"
+    const val Enabled = "RECONCILER_FULL_ENABLED"
 
     /**
      * Run interval for the full reconciliation process.

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -29,7 +29,6 @@ services:
     labels:
     - "com.centurylinklabs.watchtower.enable=${VDI_POSTGRES_WATCHTOWER:-false}"
 
-
   kafka:
     image: veupathdb/apache-kafka:${VDI_KAFKA_TAG:-latest}
     healthcheck:
@@ -123,12 +122,18 @@ services:
       UPDATE_META_HANDLER_WORKER_POOL_SIZE: ${UPDATE_META_HANDLER_WORKER_POOL_SIZE}
       UPDATE_META_HANDLER_WORK_QUEUE_SIZE: ${UPDATE_META_HANDLER_WORK_QUEUE_SIZE}
 
+      # Reconciliation Handler
+      RECONCILIATION_HANDLER_KAFKA_CONSUMER_CLIENT_ID: ${RECONCILIATION_HANDLER_KAFKA_CONSUMER_CLIENT_ID}
+      RECONCILIATION_HANDLER_WORKER_POOL_SIZE: ${RECONCILIATION_HANDLER_WORKER_POOL_SIZE}
+      RECONCILIATION_HANDLER_WORK_QUEUE_SIZE: ${RECONCILIATION_HANDLER_WORK_QUEUE_SIZE}
+
       # Cache DB
       CACHE_DB_USERNAME: ${CACHE_DB_USERNAME:?}
       CACHE_DB_PASSWORD: ${CACHE_DB_PASSWORD:?}
       CACHE_DB_NAME: ${CACHE_DB_NAME:?}
       CACHE_DB_HOST: ${CACHE_DB_HOST:?}
-      CACHE_DB_PORT: ${CACHE_DB_PORT:-5432}
+      CACHE_DB_PORT: ${CACHE_DB_PORT}
+      CACHE_DB_POOL_SIZE: ${CACHE_DB_POOL_SIZE}
 
       # Kafka
       KAFKA_SERVERS: ${KAFKA_SERVERS:?}
@@ -141,13 +146,15 @@ services:
       KAFKA_TOPIC_SHARE_TRIGGERS: ${KAFKA_TOPIC_SHARE_TRIGGERS}
       KAFKA_TOPIC_SOFT_DELETE_TRIGGERS: ${KAFKA_TOPIC_SOFT_DELETE_TRIGGERS}
       KAFKA_TOPIC_UPDATE_META_TRIGGERS: ${KAFKA_TOPIC_UPDATE_META_TRIGGERS}
+      KAFKA_TOPIC_RECONCILIATION_TRIGGERS: ${KAFKA_TOPIC_RECONCILIATION_TRIGGERS}
 
       KAFKA_MESSAGE_KEY_HARD_DELETE_TRIGGERS: ${KAFKA_MESSAGE_KEY_HARD_DELETE_TRIGGERS}
       KAFKA_MESSAGE_KEY_IMPORT_TRIGGERS: ${KAFKA_MESSAGE_KEY_IMPORT_TRIGGERS}
       KAFKA_MESSAGE_KEY_INSTALL_TRIGGERS: ${KAFKA_MESSAGE_KEY_INSTALL_TRIGGERS}
       KAFKA_MESSAGE_KEY_SHARE_TRIGGERS: ${KAFKA_MESSAGE_KEY_SHARE_TRIGGERS}
       KAFKA_MESSAGE_KEY_SOFT_DELETE_TRIGGERS: ${KAFKA_MESSAGE_KEY_SOFT_DELETE_TRIGGERS}
       KAFKA_MESSAGE_KEY_UPDATE_META_TRIGGERS: ${KAFKA_MESSAGE_KEY_UPDATE_META_TRIGGERS}
+      KAFKA_MESSAGE_KEY_RECONCILIATION_TRIGGERS: ${KAFKA_MESSAGE_KEY_RECONCILIATION_TRIGGERS}
 
       # RabbitMQ
       GLOBAL_RABBIT_HOST: ${GLOBAL_RABBIT_HOST:?}
@@ -178,7 +185,7 @@ services:
       S3_BUCKET_NAME: ${S3_BUCKET_NAME:?}
 
       # Reconciler
-      RECONCILER_ENABLED: ${RECONCILER_ENABLED}
+      RECONCILER_FULL_ENABLED: ${RECONCILER_FULL_ENABLED}
       RECONCILER_FULL_RUN_INTERVAL: ${RECONCILER_FULL_RUN_INTERVAL}
       RECONCILER_SLIM_RUN_INTERVAL: ${RECONCILER_SLIM_RUN_INTERVAL}
 
@@ -379,6 +386,12 @@ services:
       SITE_BUILD: ${SITE_BUILD:?}
       DATASET_INSTALL_ROOT: ${DATASET_DIRECTORY_TARGET_PATH:?}
 
+      IMPORT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_NOOP_IMPORT_SCRIPT_MAX_DURATION}
+      INSTALL_DATA_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_NOOP_INSTALL_DATA_SCRIPT_MAX_DURATION}
+      INSTALL_META_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_NOOP_INSTALL_META_SCRIPT_MAX_DURATION}
+      UNINSTALL_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_NOOP_UNINSTALL_SCRIPT_MAX_DURATION}
+      CHECK_COMPAT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_NOOP_CHECK_COMPAT_SCRIPT_MAX_DURATION}
+
       DB_CONNECTION_ENABLED_AMOEBA: ${DB_CONNECTION_ENABLED_AMOEBA}
       DB_CONNECTION_NAME_AMOEBA: ${DB_CONNECTION_NAME_AMOEBA:?}
       DB_CONNECTION_LDAP_AMOEBA: ${DB_CONNECTION_LDAP_AMOEBA:?}
@@ -518,6 +531,12 @@ services:
       SITE_BUILD: ${SITE_BUILD:?}
       DATASET_INSTALL_ROOT: ${DATASET_DIRECTORY_TARGET_PATH:?}
 
+      IMPORT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_GENELIST_IMPORT_SCRIPT_MAX_DURATION}
+      INSTALL_DATA_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_GENELIST_INSTALL_DATA_SCRIPT_MAX_DURATION}
+      INSTALL_META_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_GENELIST_INSTALL_META_SCRIPT_MAX_DURATION}
+      UNINSTALL_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_GENELIST_UNINSTALL_SCRIPT_MAX_DURATION}
+      CHECK_COMPAT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_GENELIST_CHECK_COMPAT_SCRIPT_MAX_DURATION}
+
       DB_CONNECTION_ENABLED_AMOEBA: ${DB_CONNECTION_ENABLED_AMOEBA}
       DB_CONNECTION_NAME_AMOEBA: ${DB_CONNECTION_NAME_AMOEBA:?}
       DB_CONNECTION_LDAP_AMOEBA: ${DB_CONNECTION_LDAP_AMOEBA:?}
@@ -657,7 +676,11 @@ services:
       SITE_BUILD: ${SITE_BUILD:?}
       DATASET_INSTALL_ROOT: ${DATASET_DIRECTORY_TARGET_PATH:?}
 
-      IMPORT_SCRIPT_MAX_DURATION: 6h
+      IMPORT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_ISASIMPLE_IMPORT_SCRIPT_MAX_DURATION}
+      INSTALL_DATA_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_ISASIMPLE_INSTALL_DATA_SCRIPT_MAX_DURATION}
+      INSTALL_META_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_ISASIMPLE_INSTALL_META_SCRIPT_MAX_DURATION}
+      UNINSTALL_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_ISASIMPLE_UNINSTALL_SCRIPT_MAX_DURATION}
+      CHECK_COMPAT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_ISASIMPLE_CHECK_COMPAT_SCRIPT_MAX_DURATION}
 
       DB_CONNECTION_ENABLED_AMOEBA: ${DB_CONNECTION_ENABLED_AMOEBA}
       DB_CONNECTION_NAME_AMOEBA: ${DB_CONNECTION_NAME_AMOEBA:?}
@@ -798,6 +821,12 @@ services:
       SITE_BUILD: ${SITE_BUILD:?}
       DATASET_INSTALL_ROOT: ${DATASET_DIRECTORY_TARGET_PATH:?}
 
+      IMPORT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIGWIG_IMPORT_SCRIPT_MAX_DURATION}
+      INSTALL_DATA_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIGWIG_INSTALL_DATA_SCRIPT_MAX_DURATION}
+      INSTALL_META_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIGWIG_INSTALL_META_SCRIPT_MAX_DURATION}
+      UNINSTALL_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIGWIG_UNINSTALL_SCRIPT_MAX_DURATION}
+      CHECK_COMPAT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIGWIG_CHECK_COMPAT_SCRIPT_MAX_DURATION}
+
       DB_CONNECTION_ENABLED_AMOEBA: ${DB_CONNECTION_ENABLED_AMOEBA}
       DB_CONNECTION_NAME_AMOEBA: ${DB_CONNECTION_NAME_AMOEBA:?}
       DB_CONNECTION_LDAP_AMOEBA: ${DB_CONNECTION_LDAP_AMOEBA:?}
@@ -937,6 +966,12 @@ services:
       SITE_BUILD: ${SITE_BUILD:?}
       DATASET_INSTALL_ROOT: ${DATASET_DIRECTORY_TARGET_PATH:?}
 
+      IMPORT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIOM_IMPORT_SCRIPT_MAX_DURATION}
+      INSTALL_DATA_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIOM_INSTALL_DATA_SCRIPT_MAX_DURATION}
+      INSTALL_META_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIOM_INSTALL_META_SCRIPT_MAX_DURATION}
+      UNINSTALL_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIOM_UNINSTALL_SCRIPT_MAX_DURATION}
+      CHECK_COMPAT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_BIOM_CHECK_COMPAT_SCRIPT_MAX_DURATION}
+
       DB_CONNECTION_ENABLED_AMOEBA: ${DB_CONNECTION_ENABLED_AMOEBA}
       DB_CONNECTION_NAME_AMOEBA: ${DB_CONNECTION_NAME_AMOEBA:?}
       DB_CONNECTION_LDAP_AMOEBA: ${DB_CONNECTION_LDAP_AMOEBA:?}
@@ -1076,6 +1111,12 @@ services:
       SITE_BUILD: ${SITE_BUILD:?}
       DATASET_INSTALL_ROOT: ${DATASET_DIRECTORY_TARGET_PATH:?}
 
+      IMPORT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_RNASEQ_IMPORT_SCRIPT_MAX_DURATION}
+      INSTALL_DATA_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_RNASEQ_INSTALL_DATA_SCRIPT_MAX_DURATION}
+      INSTALL_META_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_RNASEQ_INSTALL_META_SCRIPT_MAX_DURATION}
+      UNINSTALL_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_RNASEQ_UNINSTALL_SCRIPT_MAX_DURATION}
+      CHECK_COMPAT_SCRIPT_MAX_DURATION: ${PLUGIN_HANDLER_RNASEQ_CHECK_COMPAT_SCRIPT_MAX_DURATION}
+
       DB_CONNECTION_ENABLED_AMOEBA: ${DB_CONNECTION_ENABLED_AMOEBA}
       DB_CONNECTION_NAME_AMOEBA: ${DB_CONNECTION_NAME_AMOEBA:?}
       DB_CONNECTION_LDAP_AMOEBA: ${DB_CONNECTION_LDAP_AMOEBA:?}