diff --git a/README.md b/README.md index 2f2f44e..c1ba8b5 100644 --- a/README.md +++ b/README.md @@ -273,4 +273,4 @@ The playbook, **collect_srvDebugFiles.yaml**, is used to automate the informatio If we only want to collect files from certain server types or server hosts, it can be achieved using the following 2 variables: * **srv_types**: you can put all required server types in a list, such as [zookeeper[,bookkeeper[,broker]]] * if not specified, all server types are collected -* **srvhost_ptn**: this can further limit the server host list if only the server host name matches certain pattern. +* **srvhost_ptn**: this can further limit the server host list if only the server host name matches certain pattern. \ No newline at end of file diff --git a/georep_getClstrClntCnf.yaml b/georep_getClstrClntCnf.yaml index 0753e75..656ddf7 100644 --- a/georep_getClstrClntCnf.yaml +++ b/georep_getClstrClntCnf.yaml @@ -17,5 +17,4 @@ with_items: - "{{ pulsar_client_config_file }}" - "{{ tgt_jwt_dest_homedir }}/token/{{ brkr_super_user_roles[0] }}.jwt" - - "{{ tgt_tls_dest_homedir }}/rootca/{{ srv_component }}_{{ public_cacert_name }}" - \ No newline at end of file + - "{{ tgt_tls_dest_homedir }}/rootca/{{ srv_component }}_{{ public_cacert_name }}" \ No newline at end of file diff --git a/group_vars/adminConsole/all b/group_vars/adminConsole/all index c8be313..c863c8e 100644 --- a/group_vars/adminConsole/all +++ b/group_vars/adminConsole/all @@ -32,7 +32,7 @@ nodejs_download_url: "{{ nodejs_download_url_base }}/{{ nodejs_bin_name }}" # # - Target package and binary directories tgt_pkg_adminConsole_dir: "{{ tgt_pkg_homedir }}/pulsar_admin_console" -tgt_adminConsole_inst_dir: /opt/pulsarAdminConsole +tgt_adminConsole_inst_dir: "/opt/pulsarAdminConsole" tgt_adminConsole_sec_dir: "{{ tgt_adminConsole_inst_dir }}/security" # - AdminConsole listending port (DO NOT change it. This is a constant) diff --git a/group_vars/all b/group_vars/all index 3f9e52b..6934cd0 100755 --- a/group_vars/all +++ b/group_vars/all @@ -62,6 +62,7 @@ pulsarLS_bin_download_url: "{{ pulsarLS_download_url_base }}/ls{{ pulsarLS_ver_m # - Local fodler that has local_bin_homedir: "~/DataStax/pulsar_bin/releases/luna_streaming" + # # Target directories # ------------------------------------------- @@ -197,6 +198,7 @@ force_tgt_jwt_token: false # - Whehter or not to force replacing the existing Pulsar server side TLS certificate files force_tgt_tls_certs: false + # # >>> Broker related security: JWT token and in-transit TLS encryption # diff --git a/group_vars/bookkeeper/all b/group_vars/bookkeeper/all index 1b1bb7a..1060dab 100644 --- a/group_vars/bookkeeper/all +++ b/group_vars/bookkeeper/all @@ -111,24 +111,13 @@ misc_bookie_config_dict: { , maxPendingReadRequestsPerThread: 10000 # - no autorecovery - , autoRecoveryDaemonEnabled: 'false' # default true -} + , autoRecoveryDaemonEnabled: 'false' -# Sometimes espeically with custom build, there might be some settings -# not generally available in the official Pulsar build yet. Put these -# settings here as experimental features -# -# NOTE: -# - You can explicitly DISABLE this feature in the Pulsar builds that -# does't have these experimental settings. -# - Or you can leave as is which is simply no-op and won't have any -# impact to the Pulsar cluster -use_bookie_experimental_feature: true - -misc_bookie_config_dict_experimental: { - # - Back Pressure - maxAddsInProgressLimit: 10000 + # Sometimes especially with custom build, there might be some settings + # not generally available in the official Pulsar build yet. Put these + # settings below as experimental features. For builds that don't support + , maxAddsInProgressLimit: 10000 , maxReadsInProgressLimit: 10000 , closeChannelOnResponseTimeout: 'true' , waitTimeoutOnResponseBackpressureMs: 100 -} +} \ No newline at end of file diff --git a/group_vars/broker/all b/group_vars/broker/all index 7b76f79..bd294fd 100644 --- a/group_vars/broker/all +++ b/group_vars/broker/all @@ -5,6 +5,7 @@ broker_conf_file: "{{ tgt_pulsar_inst_dir }}/conf/broker.conf" broker_func_worker_file: "{{ tgt_pulsar_inst_dir }}/conf/functions_worker.yml" pulsar_mem_broker: "-Xms4g -Xmx4g -XX:MaxDirectMemorySize=8g" +#pulsar_mem_broker: "-Xms5g -Xmx5g -XX:MaxDirectMemorySize=5g" broker_svc_port: 6650 broker_svc_port_tls: 6651 @@ -48,7 +49,6 @@ msg_filter_pkg_download_url_base: "https://github.com/datastax/pulsar-jms/releas msg_filter_pkg_download_url: "{{ msg_filter_pkg_download_url_base }}/{{ msg_filter_package_version }}/{{ msg_filter_package_name }}" - # # Other configuration items # ------------------------------------------- @@ -96,8 +96,8 @@ msg_filter_pkg_download_url: "{{ msg_filter_pkg_download_url_base }}/{{ msg_filt misc_broker_config_dict: { forceDeleteTenantAllowed: 'false' # default false , forceDeleteNamespaceAllowed: 'false' # default false - , allowAutoTopicCreation: 'false' # default false - , allowAutoSubscriptionCreation: 'false' # default false + , allowAutoTopicCreation: 'false' # default false + , allowAutoSubscriptionCreation: 'false' # default false , brokerDeleteInactiveTopicsEnabled: 'false' # default true , topicFencingTimeoutSeconds: '5' # default 0 , webSocketServiceEnabled: 'true' # default false @@ -115,21 +115,12 @@ misc_broker_config_dict: { , exposeProducerLevelMetricsInPrometheus: 'false' , exposeManagedCursorMetricsInPrometheus: 'false' , loadBalancerNamespaceBundleMaxSessions: 0 -} - -# Sometimes espeically with custom build, there might be some settings -# not generally available in the official Pulsar build yet. Put these -# settings here as experimental features -# -# NOTE: -# - You can explicitly DISABLE this feature in the Pulsar builds that -# does't have these experimental settings. -# - Or you can leave as is which is simply no-op and won't have any -# impact to the Pulsar cluster -use_broker_experimental_feature: true - -misc_broker_config_dict_experimental: { - cacheEvictionByMarkDeletedPosition: 'false' - , bookkeeper_waitTimeoutOnBackpressureMs: 50 , dispatchThrottlingForFilteredEntriesEnabled: 'true' + # Sometimes especially with custom build, there might be some settings + # not generally available in the official Pulsar build yet. Put these + # settings below as experimental features. For builds that don't support + # these settings, just leave them as is and they won't have any impact + , cacheEvictionByMarkDeletedPosition: 'false' + , bookkeeper_waitTimeoutOnBackpressureMs: 50 + , numWorkerThreadsForNonPersistentTopic: 32 } \ No newline at end of file diff --git a/roles/pulsar/bookie/bkie_config/_base/tasks/main.yaml b/roles/pulsar/bookie/bkie_config/_base/tasks/main.yaml index fdd1cf7..85d6e82 100755 --- a/roles/pulsar/bookie/bkie_config/_base/tasks/main.yaml +++ b/roles/pulsar/bookie/bkie_config/_base/tasks/main.yaml @@ -41,19 +41,10 @@ - { regexp: "^(#\\s?)?ensemblePlacementPolicy=", line: "ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy" } when: config_rackAwareness is defined and config_rackAwareness|bool -- name: Modify Pulsar bookkeeper config file for misc, official settings +- name: Modify Pulsar bookkeeper config file for misc settings lineinfile: path: "{{ bookie_conf_file }}" regexp: "^(#\\s?)?{{ item.key }}=" line: "{{ item.key }}={{ item.value }}" with_items: - - "{{ lookup('dict', misc_bookie_config_dict) }}" - -- name: Modify Pulsar bookkeeper config file for misc, experimental settings - lineinfile: - path: "{{ bookie_conf_file }}" - regexp: "^(#\\s?)?{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: - - "{{ lookup('dict', misc_bookie_config_dict_experimental) }}" - when: use_bookie_experimental_feature is defined and use_bookie_experimental_feature|bool \ No newline at end of file + - "{{ lookup('dict', misc_bookie_config_dict) }}" \ No newline at end of file diff --git a/roles/pulsar/broker/brkr_config/_base/tasks/main.yaml b/roles/pulsar/broker/brkr_config/_base/tasks/main.yaml index 41cf83a..aaec07a 100755 --- a/roles/pulsar/broker/brkr_config/_base/tasks/main.yaml +++ b/roles/pulsar/broker/brkr_config/_base/tasks/main.yaml @@ -108,19 +108,10 @@ line: "bookkeeperClientEnforceMinNumRacksPerWriteQuorum={{ enforceMinNumRackPerWQ|lower }}" } when: config_rackAwareness is defined and config_rackAwareness|bool -- name: Modify Pulsar broker config file for misc, official settings +- name: Modify Pulsar broker config file for misc settings lineinfile: path: "{{ broker_conf_file }}" regexp: "^(#\\s?)?{{ item.key }}=" line: "{{ item.key }}={{ item.value }}" with_items: - - "{{ lookup('dict', misc_broker_config_dict) }}" - -- name: Modify Pulsar broker config file for misc, experimental settings - lineinfile: - path: "{{ broker_conf_file }}" - regexp: "^(#\\s?)?{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: - - "{{ lookup('dict', misc_broker_config_dict_experimental) }}" - when: use_broker_experimental_feature is defined and use_broker_experimental_feature|bool \ No newline at end of file + - "{{ lookup('dict', misc_broker_config_dict) }}" \ No newline at end of file diff --git a/roles/pulsar/common/pulsar_getBinary/tasks/main.yaml b/roles/pulsar/common/pulsar_getBinary/tasks/main.yaml index 89f71df..41d049f 100755 --- a/roles/pulsar/common/pulsar_getBinary/tasks/main.yaml +++ b/roles/pulsar/common/pulsar_getBinary/tasks/main.yaml @@ -3,8 +3,8 @@ # stat: # path: "{{ tgt_pkg_pulsar_dir }}/{{ pulsarLS_bin_name }}" # register: pulsar_distro_stat -# # - debug: msg="{{pulsar_distro_stat.stat.exists }}, {{ internet_download }}" -# # when: show_debug_msg|bool +# - debug: msg="{{pulsar_distro_stat.stat.exists }}, {{ internet_download }}" +# when: show_debug_msg|bool - name: Check if the target Pulsar binary exists on the target host yet. If not, copy it from local host when 'internet_download' is false copy: diff --git a/roles/pulsar/common/pulsar_startSvc/tasks/main.yaml b/roles/pulsar/common/pulsar_startSvc/tasks/main.yaml index 8df30fa..f3aa7e8 100644 --- a/roles/pulsar/common/pulsar_startSvc/tasks/main.yaml +++ b/roles/pulsar/common/pulsar_startSvc/tasks/main.yaml @@ -16,7 +16,7 @@ # when: show_debug_msg|bool - name: Start the Pulsar server component ({{ srv_component }}), if it is not running yet # NOTE: replace "functions_worker" to "functions-worker" - shell: "{% if sudo_needed|bool %}sudo -u {{ pulsar_user }} {% endif %}{{ srv_options }} {{ tgt_pulsar_inst_dir }}/bin/pulsar-daemon start {{ srv_component | replace('_', '-') }}" + shell: "{% if sudo_needed|bool %}sudo -u {{ pulsar_user }} {% endif %}{{ srv_options|trim|replace('\n',' ') }} {{ tgt_pulsar_inst_dir }}/bin/pulsar-daemon start {{ srv_component | replace('_', '-') }}" when: not srvStarted|trim|bool - name: Wait until the Pulsar server component ({{ srv_component }}) is ready diff --git a/shutdown_adminConsole.yaml b/shutdown_adminConsole.yaml index 1f72ee9..53ebb1f 100755 --- a/shutdown_adminConsole.yaml +++ b/shutdown_adminConsole.yaml @@ -22,5 +22,4 @@ - "{{ tgt_adminConsole_inst_dir }}" - "{{ tgt_adminConsole_sec_dir }}" # - "{{ tgt_pkg_adminConsole_dir }}" - when: purge_adminConsole is defined and purge_adminConsole|bool - + when: purge_adminConsole is defined and purge_adminConsole|bool \ No newline at end of file diff --git a/shutdown_pulsarCluster.yaml b/shutdown_pulsarCluster.yaml index fc4b44b..f6e61a8 100755 --- a/shutdown_pulsarCluster.yaml +++ b/shutdown_pulsarCluster.yaml @@ -69,7 +69,7 @@ with_items: - "{{ tgt_pulsar_bookie_journal_data_homedirs }}" - "{{ tgt_pulsar_bookie_ledger_data_homedirs }}" - when: purge_pulsar is defined and purge_pulsar|bool + when: purge_pulsar is defined and purge_pulsar|bool and stop_component is not defined # Stop zookeepers - hosts: zookeeper @@ -95,7 +95,7 @@ # warn: no with_items: - "{{ tgt_pulsar_zk_data_homedir }}" - when: purge_pulsar is defined and purge_pulsar|bool + when: purge_pulsar is defined and purge_pulsar|bool and stop_component is not defined # Clean up pulsar binaries and logs - hosts: pulsarServer @@ -112,4 +112,4 @@ - "{{ tgt_pulsar_gc_log_homedir }}" - "{{ tgt_pulsar_log_homedir }}" # - "{{ tgt_pkg_dir }}" - when: purge_pulsar is defined and purge_pulsar|bool \ No newline at end of file + when: purge_pulsar is defined and purge_pulsar|bool and stop_component is not defined \ No newline at end of file