Misc minor code cleanup and fix

README.md

@@ -273,4 +273,4 @@ The playbook, **collect_srvDebugFiles.yaml**, is used to automate the informatio
 If we only want to collect files from certain server types or server hosts, it can be achieved using the following 2 variables:
 * **srv_types**: you can put all required server types in a list, such as [zookeeper[,bookkeeper[,broker]]]
   * if not specified, all server types are collected
-* **srvhost_ptn**: this can further limit the server host list if only the server host name matches certain pattern.
+* **srvhost_ptn**: this can further limit the server host list if only the server host name matches certain pattern.

georep_getClstrClntCnf.yaml

@@ -17,5 +17,4 @@
       with_items:
         - "{{ pulsar_client_config_file }}"
         - "{{ tgt_jwt_dest_homedir }}/token/{{ brkr_super_user_roles[0] }}.jwt"
-        - "{{ tgt_tls_dest_homedir }}/rootca/{{ srv_component }}_{{ public_cacert_name }}"
-
+        - "{{ tgt_tls_dest_homedir }}/rootca/{{ srv_component }}_{{ public_cacert_name }}"

group_vars/adminConsole/all

@@ -32,7 +32,7 @@ nodejs_download_url: "{{ nodejs_download_url_base }}/{{ nodejs_bin_name }}"
 #
 # - Target package and binary directories
 tgt_pkg_adminConsole_dir: "{{ tgt_pkg_homedir }}/pulsar_admin_console"
-tgt_adminConsole_inst_dir: /opt/pulsarAdminConsole
+tgt_adminConsole_inst_dir: "/opt/pulsarAdminConsole"
 tgt_adminConsole_sec_dir: "{{ tgt_adminConsole_inst_dir }}/security"
 
 # - AdminConsole listending port (DO NOT change it. This is a constant)

group_vars/all

@@ -62,6 +62,7 @@ pulsarLS_bin_download_url: "{{ pulsarLS_download_url_base }}/ls{{ pulsarLS_ver_m
 # - Local fodler that has 
 local_bin_homedir: "~/DataStax/pulsar_bin/releases/luna_streaming"
 
+
 #
 # Target directories
 # -------------------------------------------
@@ -197,6 +198,7 @@ force_tgt_jwt_token: false
 # - Whehter or not to force replacing the existing Pulsar server side TLS certificate files
 force_tgt_tls_certs: false
 
+
 #
 # >>> Broker related security: JWT token and in-transit TLS encryption
 #

group_vars/bookkeeper/all

@@ -111,24 +111,13 @@ misc_bookie_config_dict: {
   , maxPendingReadRequestsPerThread: 10000
 
   # - no autorecovery
-  , autoRecoveryDaemonEnabled: 'false'   # default true
-}
+  , autoRecoveryDaemonEnabled: 'false'
 
-# Sometimes espeically with custom build, there might be some settings
-# not generally available in the official Pulsar build yet. Put these
-# settings here as experimental features
-#
-# NOTE: 
-# - You can explicitly DISABLE this feature in the Pulsar builds that 
-#   does't have these experimental settings. 
-# - Or you can leave as is which is simply no-op and won't have any 
-#   impact to  the Pulsar cluster
-use_bookie_experimental_feature: true
-
-misc_bookie_config_dict_experimental: {
-  # - Back Pressure
-  maxAddsInProgressLimit: 10000
+  # Sometimes especially with custom build, there might be some settings
+  # not generally available in the official Pulsar build yet. Put these
+  # settings below as experimental features. For builds that don't support
+  , maxAddsInProgressLimit: 10000
   , maxReadsInProgressLimit: 10000
   , closeChannelOnResponseTimeout: 'true'
   , waitTimeoutOnResponseBackpressureMs: 100
-}
+}

group_vars/broker/all

@@ -5,6 +5,7 @@
 broker_conf_file: "{{ tgt_pulsar_inst_dir }}/conf/broker.conf"
 broker_func_worker_file: "{{ tgt_pulsar_inst_dir }}/conf/functions_worker.yml"
 pulsar_mem_broker: "-Xms4g -Xmx4g -XX:MaxDirectMemorySize=8g"
+#pulsar_mem_broker: "-Xms5g -Xmx5g -XX:MaxDirectMemorySize=5g"
 
 broker_svc_port: 6650
 broker_svc_port_tls: 6651
@@ -48,7 +49,6 @@ msg_filter_pkg_download_url_base: "https://github.com/datastax/pulsar-jms/releas
 msg_filter_pkg_download_url: "{{ msg_filter_pkg_download_url_base }}/{{ msg_filter_package_version }}/{{ msg_filter_package_name }}"
 
 
-
 #
 # Other configuration items
 # -------------------------------------------
@@ -96,8 +96,8 @@ msg_filter_pkg_download_url: "{{ msg_filter_pkg_download_url_base }}/{{ msg_filt
 misc_broker_config_dict: {
   forceDeleteTenantAllowed: 'false'             # default false
   , forceDeleteNamespaceAllowed: 'false'        # default false
-  , allowAutoTopicCreation: 'false'              # default false
-  , allowAutoSubscriptionCreation: 'false'       # default false
+  , allowAutoTopicCreation: 'false'             # default false
+  , allowAutoSubscriptionCreation: 'false'      # default false
   , brokerDeleteInactiveTopicsEnabled: 'false'  # default true 
   , topicFencingTimeoutSeconds: '5'             # default 0
   , webSocketServiceEnabled: 'true'             # default false
@@ -115,21 +115,12 @@ misc_broker_config_dict: {
   , exposeProducerLevelMetricsInPrometheus: 'false'
   , exposeManagedCursorMetricsInPrometheus: 'false'
   , loadBalancerNamespaceBundleMaxSessions: 0
-}
-
-# Sometimes espeically with custom build, there might be some settings
-# not generally available in the official Pulsar build yet. Put these
-# settings here as experimental features
-#
-# NOTE: 
-# - You can explicitly DISABLE this feature in the Pulsar builds that 
-#   does't have these experimental settings. 
-# - Or you can leave as is which is simply no-op and won't have any 
-#   impact to  the Pulsar cluster
-use_broker_experimental_feature: true
-
-misc_broker_config_dict_experimental: {
-  cacheEvictionByMarkDeletedPosition: 'false'
-  , bookkeeper_waitTimeoutOnBackpressureMs: 50 
   , dispatchThrottlingForFilteredEntriesEnabled: 'true'
+  # Sometimes especially with custom build, there might be some settings
+  # not generally available in the official Pulsar build yet. Put these
+  # settings below as experimental features. For builds that don't support
+  # these settings, just leave them as is and they won't have any impact
+  , cacheEvictionByMarkDeletedPosition: 'false'
+  , bookkeeper_waitTimeoutOnBackpressureMs: 50 
+  , numWorkerThreadsForNonPersistentTopic: 32
 }

roles/pulsar/bookie/bkie_config/_base/tasks/main.yaml

@@ -41,19 +41,10 @@
     - { regexp: "^(#\\s?)?ensemblePlacementPolicy=", line: "ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy" }
   when: config_rackAwareness is defined and config_rackAwareness|bool
 
-- name: Modify Pulsar bookkeeper config file for misc, official settings
+- name: Modify Pulsar bookkeeper config file for misc settings
   lineinfile:
     path: "{{ bookie_conf_file }}"
     regexp: "^(#\\s?)?{{ item.key }}="
     line: "{{ item.key }}={{ item.value }}"
   with_items:
-    - "{{ lookup('dict', misc_bookie_config_dict) }}"
-
-- name: Modify Pulsar bookkeeper config file for misc, experimental settings
-  lineinfile:
-    path: "{{ bookie_conf_file }}"
-    regexp: "^(#\\s?)?{{ item.key }}="
-    line: "{{ item.key }}={{ item.value }}"
-  with_items:
-    - "{{ lookup('dict', misc_bookie_config_dict_experimental) }}"
-  when: use_bookie_experimental_feature is defined and use_bookie_experimental_feature|bool
+    - "{{ lookup('dict', misc_bookie_config_dict) }}"

roles/pulsar/broker/brkr_config/_base/tasks/main.yaml

@@ -108,19 +108,10 @@
         line: "bookkeeperClientEnforceMinNumRacksPerWriteQuorum={{ enforceMinNumRackPerWQ|lower }}" }
   when: config_rackAwareness is defined and config_rackAwareness|bool
 
-- name: Modify Pulsar broker config file for misc, official settings
+- name: Modify Pulsar broker config file for misc settings
   lineinfile:
     path: "{{ broker_conf_file }}"
     regexp: "^(#\\s?)?{{ item.key }}="
     line: "{{ item.key }}={{ item.value }}"
   with_items:
-    - "{{ lookup('dict', misc_broker_config_dict) }}"
-
-- name: Modify Pulsar broker config file for misc, experimental settings 
-  lineinfile:
-    path: "{{ broker_conf_file }}"
-    regexp: "^(#\\s?)?{{ item.key }}="
-    line: "{{ item.key }}={{ item.value }}"
-  with_items:
-    - "{{ lookup('dict', misc_broker_config_dict_experimental) }}"
-  when: use_broker_experimental_feature is defined and use_broker_experimental_feature|bool
+    - "{{ lookup('dict', misc_broker_config_dict) }}"

roles/pulsar/common/pulsar_getBinary/tasks/main.yaml

@@ -3,8 +3,8 @@
 #   stat:
 #     path: "{{ tgt_pkg_pulsar_dir }}/{{ pulsarLS_bin_name }}"
 #   register: pulsar_distro_stat
-# # - debug: msg="{{pulsar_distro_stat.stat.exists }}, {{ internet_download }}"
-# #   when: show_debug_msg|bool
+# - debug: msg="{{pulsar_distro_stat.stat.exists }}, {{ internet_download }}"
+#   when: show_debug_msg|bool
 
 - name: Check if the target Pulsar binary exists on the target host yet. If not, copy it from local host when 'internet_download' is false
   copy:

roles/pulsar/common/pulsar_startSvc/tasks/main.yaml

@@ -16,7 +16,7 @@
 #   when: show_debug_msg|bool
 - name: Start the Pulsar server component ({{ srv_component }}), if it is not running yet
   # NOTE: replace "functions_worker" to "functions-worker"
-  shell: "{% if sudo_needed|bool %}sudo -u {{ pulsar_user }} {% endif %}{{ srv_options }} {{ tgt_pulsar_inst_dir }}/bin/pulsar-daemon start {{ srv_component | replace('_', '-') }}"
+  shell: "{% if sudo_needed|bool %}sudo -u {{ pulsar_user }} {% endif %}{{ srv_options|trim|replace('\n',' ') }} {{ tgt_pulsar_inst_dir }}/bin/pulsar-daemon start {{ srv_component | replace('_', '-') }}"
   when: not srvStarted|trim|bool
 
 - name: Wait until the Pulsar server component ({{ srv_component }}) is ready

shutdown_adminConsole.yaml

@@ -22,5 +22,4 @@
         - "{{ tgt_adminConsole_inst_dir }}"
         - "{{ tgt_adminConsole_sec_dir }}"
         # - "{{ tgt_pkg_adminConsole_dir }}"
-      when: purge_adminConsole is defined and purge_adminConsole|bool
-
+      when: purge_adminConsole is defined and purge_adminConsole|bool

shutdown_pulsarCluster.yaml

@@ -69,7 +69,7 @@
       with_items:
         - "{{ tgt_pulsar_bookie_journal_data_homedirs }}"
         - "{{ tgt_pulsar_bookie_ledger_data_homedirs }}"
-      when: purge_pulsar is defined and purge_pulsar|bool
+      when: purge_pulsar is defined and purge_pulsar|bool and stop_component is not defined
 
 # Stop zookeepers 
 - hosts: zookeeper
@@ -95,7 +95,7 @@
       #   warn: no
       with_items:
         - "{{ tgt_pulsar_zk_data_homedir }}"
-      when: purge_pulsar is defined and purge_pulsar|bool 
+      when: purge_pulsar is defined and purge_pulsar|bool and stop_component is not defined
 
 # Clean up pulsar binaries and logs
 - hosts: pulsarServer
@@ -112,4 +112,4 @@
       - "{{ tgt_pulsar_gc_log_homedir }}"
       - "{{ tgt_pulsar_log_homedir }}"
       # - "{{ tgt_pkg_dir }}"
-    when: purge_pulsar is defined and purge_pulsar|bool
+    when: purge_pulsar is defined and purge_pulsar|bool and stop_component is not defined