Various fixes

1) JVM setting adjustment - spearate globally applicalbe JVM settings from component specific settings
2) Add option for using broker-style or bookie-style metadata service URL formats
3) Separate experimental broker and bookkeeper configuration settings from the settings that are generally available
4) Start and stop by specific server components

02.deploy_pulsarCluster.yaml

@@ -118,7 +118,7 @@
     # Check the current status of zookeepers
     - { role: misc/_check_svc_status, when: deploy_status=="add" }
     # Start zookepers
-    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ zk_jvm_options }}", when: deploy_status=="add" }
+    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ zk_jvm_options|trim|replace('\n',' ') }}", when: deploy_status=="add" }
 
 # - Initialzie Pulsar cluster metadata after zookeepers are up and running
 - hosts: zookeeper
@@ -127,7 +127,7 @@
   become_method: sudo
   run_once: true
   roles: 
-    - { role: pulsar/zookeeper/zkpr_initMetadata, srv_options: "{{ zk_jvm_options }}", when: isScaling|bool }
+    - { role: pulsar/zookeeper/zkpr_initMetadata, srv_options: "{{ zk_jvm_options_b|trim|replace('\n',' ') }}", when: isScaling|bool }
 
 
 #########################
@@ -150,7 +150,7 @@
     # Check the current status of bookies
     - { role: misc/_check_svc_status, when: deploy_status=="add" }
     # Start bookies
-    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ bookie_jvm_options }}", when: deploy_status=="add" }
+    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ bookie_jvm_options|trim|replace('\n',' ') }}", when: deploy_status=="add" }
 
 # - Sanity test of whether bookies are up and running properly
 - hosts: bookkeeper
@@ -159,7 +159,7 @@
   become_method: sudo
   run_once: true
   roles:
-    - { role: pulsar/bookie/bkie_sanityCheck, srv_options: "{{ bookie_jvm_options_b }}", when: isScaling|bool }
+    - { role: pulsar/bookie/bkie_sanityCheck, srv_options: "{{ bookie_jvm_options_b|trim|replace('\n',' ') }}", when: isScaling|bool }
 
 
 #########################
@@ -188,7 +188,7 @@
     # Check the current status of brokers
     - { role: misc/_check_svc_status, when: deploy_status=="add" }
     # Start brokers
-    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ broker_jvm_options }}", when: deploy_status=="add" }
+    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ broker_jvm_options|trim|replace('\n',' ') }}", when: deploy_status=="add" }
 
 # - Initialzie transaction coordinator metadata when needed
 - hosts: broker
@@ -197,7 +197,7 @@
   become_method: sudo
   run_once: true
   roles: 
-    - { role: pulsar/broker/brkr_initTransMeta, srv_options: "{{ broker_jvm_options }}", 
+    - { role: pulsar/broker/brkr_initTransMeta, srv_options: "{{ broker_jvm_options|trim|replace('\n',' ') }}", 
         when: enable_transaction|bool and deploy_status=="add" }
 
 

04.deploy_pulsarFuncWorker.yaml

@@ -66,4 +66,4 @@
     # Check the current status of functions worker
     - { role: misc/_check_svc_status }
     # Start functions worker
-    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ funcs_worker_jvm_options }}" }
+    - { role: pulsar/common/pulsar_startSvc, srv_options: "{{ funcs_worker_jvm_options|trim|replace('\n',' ') }}" }

group_vars/all

@@ -141,6 +141,22 @@ tgt_collected_srvfiles_homedir: /opt/pulsar_srvfiles_collection
 local_collected_srvfiles_homedir: collected_srv_files
 
 
+#
+# Misc settigns 
+# -------------------------------------------
+#
+# commong JVM settings that applies to all Pulsar server components
+common_jvm_settings: |
+  PULSAR_EXTRA_OPTS="-XX:+PerfDisableSharedMem {{ component_pulsar_extra_opts | default('') }}"
+  PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+HeapDumpOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError {{ component_pulsar_gc | default('') }}"
+  PULSAR_GC_LOG="-Xlog:gc*,safepoint:{{ tgt_pulsar_gc_log_homedir }}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
+
+# Use which version as the zookeeper metadata server. Valid opions: 
+# true:  use the (default) format for broker
+# false: use the format for bookkeeper
+zk_metadata_srv_format_broker: "false"
+
+
 #
 # Pulsar security settings
 # -------------------------------------------
@@ -189,7 +205,7 @@ force_tgt_tls_certs: false
 # - Wether or not to enable Pulsar JWT authentication and authorization
 enable_brkr_authNZ: true
 # - Whether or not to enable Pulsar In-Transit TLS encryption
-enable_brkr_tls: false
+enable_brkr_tls: true
 
 enable_brkr_security: "{{ (enable_brkr_authNZ is defined and enable_brkr_authNZ|bool) or (enable_brkr_tls is defined and enable_brkr_tls|bool) }}"
 
@@ -249,7 +265,7 @@ fwrkr_key_password: "MyFwrkrSrvCertPass"
 #
 #
 # - Wether or not to enable AdminConsole HTTPS
-enable_ac_https: false
+enable_ac_https: true
 # - Whether or not to skip generating Pulsar functions worker TLS certificate files on the local host
 skip_ac_tls_certs_generatation: false
 # - Whehter or not to force replacing the existing AdminConsole server side TLS certificate files

group_vars/bookkeeper/all

@@ -4,6 +4,7 @@
 #
 bookie_conf_file: "{{ tgt_pulsar_inst_dir }}/conf/bookkeeper.conf"
 pulsar_mem_bookie: "-Xms4g -Xmx4g -XX:MaxDirectMemorySize=4g"
+pulsar_mem_bookie_b: "-Xms1g -Xmx1g"
 
 bookie_listening_port: 3181
 
@@ -20,27 +21,27 @@ dft_bookie_ledger_data_homedirs:
 # Non-default journal and ledger data directories (when "customize_datadir" is true)
 # - Multiple directories are supported 
 cust_bookie_jorunal_data_homedirs:
-  - /var/lib/pulsar/bookie/journal/data1
-  - /var/lib/pulsar/bookie/journal/data2 
+  - "/var/lib/pulsar/bookie/journal/data1"
+  - "/var/lib/pulsar/bookie/journal/data2"
 cust_bookie_ledger_data_homedirs:
-  - /var/lib/pulsar/bookie/ledger/data1
-  - /var/lib/pulsar/bookie/ledger/data2
+  - "/var/lib/pulsar/bookie/ledger/data1"
+  - "/var/lib/pulsar/bookie/ledger/data2"
 
 tgt_pulsar_bookie_journal_data_homedirs: "{% if customize_datadir is defined and customize_datadir|bool %}{{ cust_bookie_jorunal_data_homedirs }}{% else %}{{ dft_bookie_jorunal_data_homedirs }}{% endif %}"
 tgt_pulsar_bookie_ledger_data_homedirs: "{% if customize_datadir is defined and customize_datadir|bool %}{{ cust_bookie_ledger_data_homedirs }}{% else %}{{ dft_bookie_ledger_data_homedirs }}{% endif %}"
 
 
 # extra JVM options to start bookkeepers
+component_pulsar_extra_opts: ""
+component_pulsar_gc: "-XX:HeapDumpPath={{ tgt_pulsar_log_homedir }}/bookkeeper"
 bookie_jvm_options: > 
-  {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_bookie }}" {% endif %}
-  PULSAR_EXTRA_OPTS="-XX:+PerfDisableSharedMem"
-  PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10"
-  PULSAR_GC_LOG="-Xlog:gc*,safepoint:{{ tgt_pulsar_gc_log_homedir }}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
+  {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_bookie }}"{% endif %}
+  {{ common_jvm_settings }}
   PULSAR_LOG_DIR="{{ tgt_pulsar_log_homedir }}/bookkeeper"
 
-# JVM options to run bookie commands
+# JVM options to run bookie CLI commands
 bookie_jvm_options_b: > 
-  {% if customize_jvm is defined and customize_jvm|bool %}BOOKIE_MEM="{{ pulsar_mem_bookie }}" {% endif %}
+  {% if customize_jvm is defined and customize_jvm|bool %}BOOKIE_MEM="{{ pulsar_mem_bookie_b }}" {% endif %}
   BOOKIE_EXTRA_OPTS="-XX:+PerfDisableSharedMem"
   BOOKIE_GC_LOG=" "
   BOOKIE_GC="-Xlog:gc*,safepoint:{{ tgt_pulsar_gc_log_homedir }}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
@@ -104,5 +105,30 @@ misc_bookie_config_dict: {
   , logSizeLimit: 1073741824        ## for large disk capacity, use the default value
 
   # - Other performance related settings 
-  , verifyMetadataOnGC: 'true'    # default false
+  , verifyMetadataOnGC: 'true'
+
+  # - Back Pressure
+  , maxPendingReadRequestsPerThread: 10000
+
+  # - no autorecovery
+  , autoRecoveryDaemonEnabled: 'false'   # default true
+}
+
+# Sometimes espeically with custom build, there might be some settings
+# not generally available in the official Pulsar build yet. Put these
+# settings here as experimental features
+#
+# NOTE: 
+# - You can explicitly DISABLE this feature in the Pulsar builds that 
+#   does't have these experimental settings. 
+# - Or you can leave as is which is simply no-op and won't have any 
+#   impact to  the Pulsar cluster
+use_bookie_experimental_feature: true
+
+misc_bookie_config_dict_experimental: {
+  # - Back Pressure
+  maxAddsInProgressLimit: 10000
+  , maxReadsInProgressLimit: 10000
+  , closeChannelOnResponseTimeout: 'true'
+  , waitTimeoutOnResponseBackpressureMs: 100
 }

group_vars/broker/all

@@ -13,11 +13,11 @@ web_svc_port: 8080
 web_svc_port_tls: 8443
 
 # extra JVM options to start bookkeepers
+component_pulsar_extra_opts: ""
+component_pulsar_gc: "-XX:HeapDumpPath={{ tgt_pulsar_log_homedir }}/broker"
 broker_jvm_options: > 
-  {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_broker }}" {% endif %}
-  PULSAR_EXTRA_OPTS="-XX:+PerfDisableSharedMem"
-  PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10"
-  PULSAR_GC_LOG="-Xlog:gc*,safepoint:{{ tgt_pulsar_gc_log_homedir }}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
+  {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_broker }}"{% endif %}
+  {{ common_jvm_settings }}
   PULSAR_LOG_DIR="{{ tgt_pulsar_log_homedir }}/broker"
 
 # whether or not to use customized message replication settings (E/Qw/Qa)
@@ -96,17 +96,40 @@ msg_filter_pkg_download_url: "{{ msg_filter_pkg_download_url_base }}/{{ msg_filt
 misc_broker_config_dict: {
   forceDeleteTenantAllowed: 'false'             # default false
   , forceDeleteNamespaceAllowed: 'false'        # default false
-  , allowAutoTopicCreation: 'true'              # default false
-  , allowAutoSubscriptionCreation: 'true'       # default false
+  , allowAutoTopicCreation: 'false'              # default false
+  , allowAutoSubscriptionCreation: 'false'       # default false
   , brokerDeleteInactiveTopicsEnabled: 'false'  # default true 
   , topicFencingTimeoutSeconds: '5'             # default 0
   , webSocketServiceEnabled: 'true'             # default false
   , bookkeeper_opportunisticStriping: 'true'    # default false
-  # 'systemTopicEnabled' is always enabled
+  , systemTopicEnabled: 'true'
   , topicLevelPoliciesEnabled: 'true'           # default false
-  , managedLedgerCursorBackloggedThreshold: 1000
-  , managedLedgerCacheEvictionTimeThresholdMillis: 2000
-  , managedLedgerMinimumBacklogCursorsForCaching: 4
-  , managedLedgerMinimumBacklogEntriesForCaching: 1000
-  , managedLedgerMaxBacklogBetweenCursorsForCaching: 20000
+  , managedLedgerCursorBackloggedThreshold: 2147483647
+  , managedLedgerCacheEvictionTimeThresholdMillis: 10000
+  , managedLedgerMinimumBacklogCursorsForCaching: 3
+  , managedLedgerMinimumBacklogEntriesForCaching: 1
+  , managedLedgerMaxBacklogBetweenCursorsForCaching: 1000000 
+  , managedLedgerCacheEvictionFrequency: 0.2    # 1 eviction for 5 seconds
+  , managedLedgerCacheSizeMB: 2048
+  , exposeConsumerLevelMetricsInPrometheus: 'false'
+  , exposeProducerLevelMetricsInPrometheus: 'false'
+  , exposeManagedCursorMetricsInPrometheus: 'false'
+  , loadBalancerNamespaceBundleMaxSessions: 0
 }
+
+# Sometimes espeically with custom build, there might be some settings
+# not generally available in the official Pulsar build yet. Put these
+# settings here as experimental features
+#
+# NOTE: 
+# - You can explicitly DISABLE this feature in the Pulsar builds that 
+#   does't have these experimental settings. 
+# - Or you can leave as is which is simply no-op and won't have any 
+#   impact to  the Pulsar cluster
+use_broker_experimental_feature: true
+
+misc_broker_config_dict_experimental: {
+  cacheEvictionByMarkDeletedPosition: 'false'
+  , bookkeeper_waitTimeoutOnBackpressureMs: 50 
+  , dispatchThrottlingForFilteredEntriesEnabled: 'true'
+}

group_vars/functions_worker/all

@@ -10,11 +10,11 @@ funcs_worker_port: 6750
 funcs_worker_port_tls: 6751
 
 # extra JVM options to start functions worker
-funcs_worker_jvm_options: > 
+component_pulsar_extra_opts: ""
+component_pulsar_gc: "-XX:HeapDumpPath={{ tgt_pulsar_log_homedir }}/functions_worker"
+funcs_worker_jvm_options: >
   {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_funcs_worker }}" {% endif %}
-  PULSAR_EXTRA_OPTS="{{ pulsar_extra_jvm_opts }}"
-  PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10"
-  PULSAR_GC_LOG="-Xlog:gc*,safepoint:{{ tgt_pulsar_gc_log_homedir }}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
+  {{ common_jvm_settings }}
   PULSAR_LOG_DIR="{{ tgt_pulsar_log_homedir }}/functions_worker"
 
 # Function package repilca num

group_vars/zookeeper/all

@@ -4,6 +4,7 @@
 #
 zk_conf_file: "{{ tgt_pulsar_inst_dir }}/conf/zookeeper.conf"
 pulsar_mem_zk: "-Xms1g -Xmx1g -XX:MaxDirectMemorySize=2g"
+pulsar_mem_zk_b: "-Xms1g -Xmx1g"
 
 zk_clnt_port: 2181
 zk_admin_srv_enable: true
@@ -23,9 +24,15 @@ tgt_pulsar_zk_data_homedir: "{% if customize_datadir is defined and customize_da
 zk_storage_mnt_path: "{{ tgt_pulsar_zk_data_homedir }}/zookeeper"
 
 # extra JVM options to start zookeepers
-zk_jvm_options: > 
+component_pulsar_extra_opts: "-Dstats_server_port={{ zk_stats_port }}"
+component_pulsar_gc: "-XX:HeapDumpPath={{ tgt_pulsar_log_homedir }}/zookeeper"
+zk_jvm_options: >
   {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_zk }}"{% endif %}
-  PULSAR_EXTRA_OPTS="-Dstats_server_port={{ zk_stats_port }} -XX:+PerfDisableSharedMem"
-  PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10"
-  PULSAR_GC_LOG="-Xlog:gc*,safepoint:{{ tgt_pulsar_gc_log_homedir }}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
-  PULSAR_LOG_DIR="{{ tgt_pulsar_log_homedir }}/zookeeper" 
+  {{ common_jvm_settings }}
+  PULSAR_LOG_DIR="{{ tgt_pulsar_log_homedir }}/zookeeper"
+
+# JVM options to run Pulsar CLI commands
+zk_jvm_options_b: >
+  {% if customize_jvm is defined and customize_jvm|bool %}PULSAR_MEM="{{ pulsar_mem_zk_b }}"{% endif %}
+  {{ common_jvm_settings }}
+  PULSAR_LOG_DIR="{{ tgt_pulsar_log_homedir }}/zookeeper"

roles/pulsar/bookie/bkie_config/_base/tasks/main.yaml

@@ -41,10 +41,19 @@
     - { regexp: "^(#\\s?)?ensemblePlacementPolicy=", line: "ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy" }
   when: config_rackAwareness is defined and config_rackAwareness|bool
 
-- name: Modify Pulsar bookkeeper config file for misc settings
+- name: Modify Pulsar bookkeeper config file for misc, official settings
   lineinfile:
     path: "{{ bookie_conf_file }}"
     regexp: "^(#\\s?)?{{ item.key }}="
     line: "{{ item.key }}={{ item.value }}"
   with_items:
-    - "{{ lookup('dict', misc_bookie_config_dict) }}"
+    - "{{ lookup('dict', misc_bookie_config_dict) }}"
+
+- name: Modify Pulsar bookkeeper config file for misc, experimental settings
+  lineinfile:
+    path: "{{ bookie_conf_file }}"
+    regexp: "^(#\\s?)?{{ item.key }}="
+    line: "{{ item.key }}={{ item.value }}"
+  with_items:
+    - "{{ lookup('dict', misc_bookie_config_dict_experimental) }}"
+  when: use_bookie_experimental_feature is defined and use_bookie_experimental_feature|bool

roles/pulsar/bookie/bkie_config/_ver2.10/tasks/main.yaml

@@ -10,7 +10,7 @@
     regexp: "{{ item.regexp }}"
     line: "{{ item.line }}"
   with_items:
-    - { regexp: "^metadataServiceUri=", line: "metadataServiceUri={{ hostvars[groups['zookeeper'][0]]['metaSvcZkListStrVar']|trim }}" }
+    - { regexp: "^metadataServiceUri=", line: "metadataServiceUri={{ hostvars[groups['zookeeper'][0]]['metaSvcZkListStrVar2']|trim }}" }
 
 - name: Comment out old version specific bookkeeper config settings
   replace:

roles/pulsar/broker/brkr_config/_base/tasks/main.yaml

@@ -108,10 +108,19 @@
         line: "bookkeeperClientEnforceMinNumRacksPerWriteQuorum={{ enforceMinNumRackPerWQ|lower }}" }
   when: config_rackAwareness is defined and config_rackAwareness|bool
 
-- name: Modify Pulsar broker config file for misc settings
+- name: Modify Pulsar broker config file for misc, official settings
   lineinfile:
     path: "{{ broker_conf_file }}"
     regexp: "^(#\\s?)?{{ item.key }}="
     line: "{{ item.key }}={{ item.value }}"
   with_items:
-    - "{{ lookup('dict', misc_broker_config_dict) }}"
+    - "{{ lookup('dict', misc_broker_config_dict) }}"
+
+- name: Modify Pulsar broker config file for misc, experimental settings 
+  lineinfile:
+    path: "{{ broker_conf_file }}"
+    regexp: "^(#\\s?)?{{ item.key }}="
+    line: "{{ item.key }}={{ item.value }}"
+  with_items:
+    - "{{ lookup('dict', misc_broker_config_dict_experimental) }}"
+  when: use_broker_experimental_feature is defined and use_broker_experimental_feature|bool

roles/pulsar/common/pulsar_getBinary/tasks/main.yaml

@@ -1,23 +1,23 @@
 ---
-- name: Check if Pulsar binary already exists
-  stat:
-    path: "{{ tgt_pkg_pulsar_dir }}/{{ pulsarLS_bin_name }}"
-  register: pulsar_distro_stat
-# - debug: msg="{{pulsar_distro_stat.stat.exists }}, {{ internet_download }}"
-#   when: show_debug_msg|bool
+# - name: Check if Pulsar binary already exists
+#   stat:
+#     path: "{{ tgt_pkg_pulsar_dir }}/{{ pulsarLS_bin_name }}"
+#   register: pulsar_distro_stat
+# # - debug: msg="{{pulsar_distro_stat.stat.exists }}, {{ internet_download }}"
+# #   when: show_debug_msg|bool
 
 - name: Check if the target Pulsar binary exists on the target host yet. If not, copy it from local host when 'internet_download' is false
   copy:
     src: "{{ local_bin_homedir }}/{{ pulsarLS_bin_name }}"
     dest: "{{ tgt_pkg_pulsar_dir }}/{{ pulsarLS_bin_name }}"
     owner: "{{ pulsar_user }}"
     group: "{{ pulsar_user_group }}"
-  when: not pulsar_distro_stat.stat.exists and not internet_download|bool
+  when: not internet_download|bool
 
 - name: Check if the Pulsar binary exists on the target host yet. If not, Download Pulsar from internet when 'internet_download' is true 
   get_url:
     url: "{{ pulsarLS_bin_download_url }}"
     dest: "{{ tgt_pkg_pulsar_dir }}/{{ pulsarLS_bin_name }}"
     owner: "{{ pulsar_user }}"
     group: "{{ pulsar_user_group }}"
-  when: not pulsar_distro_stat.stat.exists and internet_download|bool
+  when: internet_download|bool

roles/pulsar/common/pulsar_startSvc/tasks/main.yaml

@@ -16,7 +16,7 @@
 #   when: show_debug_msg|bool
 - name: Start the Pulsar server component ({{ srv_component }}), if it is not running yet
   # NOTE: replace "functions_worker" to "functions-worker"
-  shell: "{% if sudo_needed|bool %}sudo -u {{ pulsar_user }} {% endif %}{{ srv_options|trim }} {{ tgt_pulsar_inst_dir }}/bin/pulsar-daemon start {{ srv_component | replace('_', '-') }}"
+  shell: "{% if sudo_needed|bool %}sudo -u {{ pulsar_user }} {% endif %}{{ srv_options }} {{ tgt_pulsar_inst_dir }}/bin/pulsar-daemon start {{ srv_component | replace('_', '-') }}"
   when: not srvStarted|trim|bool
 
 - name: Wait until the Pulsar server component ({{ srv_component }}) is ready