From fd4dc8ea39d934a9aae633fedfbb2e066db44df1 Mon Sep 17 00:00:00 2001 From: kaliraja Date: Wed, 18 Dec 2019 16:12:13 +0530 Subject: [PATCH] DO-1771 updated the yarn role --- ansible/roles/yarn/tasks/common.yml | 135 +++++++++++---------- ansible/roles/yarn/tasks/main.yml | 57 ++++----- ansible/roles/yarn/templates/config.j2 | 3 + ansible/roles/yarn/templates/hadoop-env.sh | 3 +- ansible/roles/yarn/templates/yarn-site.xml | 8 ++ 5 files changed, 106 insertions(+), 100 deletions(-) create mode 100644 ansible/roles/yarn/templates/config.j2 diff --git a/ansible/roles/yarn/tasks/common.yml b/ansible/roles/yarn/tasks/common.yml index f2ed89fc7a..343aa54274 100644 --- a/ansible/roles/yarn/tasks/common.yml +++ b/ansible/roles/yarn/tasks/common.yml @@ -1,73 +1,76 @@ -- name: Download and extract hadoop tarball - unarchive: - src: "{{hadoop_download_url}}" - dest: "/usr/local/" - owner: hduser - group: hadoop - creates: "{{hadoop_yarn_home}}" - remote_src: yes +- name: Common tasks for yarn master and slave + block: + - name: Download and extract hadoop tarball + unarchive: + src: "{{hadoop_download_url}}" + dest: "/usr/local/" + owner: hduser + group: hadoop + creates: "{{hadoop_yarn_home}}" + remote_src: yes -- name: Creates symlink - file: - src: /usr/local/hadoop-{{hadoop_version}} - dest: /usr/local/hadoop - owner: hduser - group: hadoop - state: link + - name: Creates symlink + file: + src: /usr/local/hadoop-{{hadoop_version}} + dest: /usr/local/hadoop + owner: hduser + group: hadoop + state: link -- name: creating conf dir - file: - path: "{{hadoop_yarn_home}}/conf" - owner: hduser - group: hadoop - recurse: yes - state: directory + - name: creating conf dir + file: + path: "{{hadoop_yarn_home}}/conf" + owner: hduser + group: hadoop + recurse: yes + state: directory -- name: Templating configs - template: - src: "{{item}}" - dest: "{{hadoop_yarn_home}}/conf/{{item}}" - owner: hduser - group: hadoop - with_items: - - yarn-site.xml - - capacity-scheduler.xml - - core-site.xml - - log4j.properties - - hadoop-env.sh + - name: Templating configs + template: + src: "{{item}}" + dest: "{{hadoop_yarn_home}}/conf/{{item}}" + owner: hduser + group: hadoop + with_items: + - yarn-site.xml + - capacity-scheduler.xml + - core-site.xml + - log4j.properties + - hadoop-env.sh -- name: Downloading artifacts - get_url: - url: "http://search.maven.org/remotecontent?filepath=org/{{item}}" - dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/" - owner: hduser - group: hadoop - with_items: - - clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar - - apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar - - apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar + - name: Downloading artifacts + get_url: + url: "http://search.maven.org/remotecontent?filepath=org/{{item}}" + dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/" + owner: hduser + group: hadoop + with_items: + - clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar + - apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar + - apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar -- name: Download and extract scala - unarchive: - src: "{{scala_download_url}}" - dest: "/usr/local/" - owner: hduser - group: hadoop - remote_src: yes + - name: Download and extract scala + unarchive: + src: "{{scala_download_url}}" + dest: "/usr/local/" + owner: hduser + group: hadoop + remote_src: yes -- name: Creates symlink - file: - src: "/usr/local/scala-{{scala_version}}" - dest: /usr/local/scala - owner: hduser - group: hadoop - state: link + - name: Creates symlink + file: + src: "/usr/local/scala-{{scala_version}}" + dest: /usr/local/scala + owner: hduser + group: hadoop + state: link -- name: copying scala files - copy: - src: "/usr/local/scala-{{scala_version}}/lib/{{item}}" - dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/" - remote_src: true - with_items: - - scala-compiler.jar - - scala-library.jar + - name: copying scala files + copy: + src: "/usr/local/scala-{{scala_version}}/lib/{{item}}" + dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/" + remote_src: true + with_items: + - scala-compiler.jar + - scala-library.jar + delegate_to: "{{slave|default(inventory_hostname)}}" diff --git a/ansible/roles/yarn/tasks/main.yml b/ansible/roles/yarn/tasks/main.yml index 52b79faa61..a297e9a951 100644 --- a/ansible/roles/yarn/tasks/main.yml +++ b/ansible/roles/yarn/tasks/main.yml @@ -12,53 +12,44 @@ - include: common.yml - lineinfile: - dest: /home/hduser/.bashrc - state: present - regexp: '^HADOOP_YARN_HOME' + dest: /home/hduser/.bashrc + state: present + regexp: '^HADOOP_YARN_HOME' line: 'HADOOP_YARN_HOME={{hadoop_yarn_home}}' - lineinfile: - dest: /home/hduser/.bashrc - state: present - regexp: '^HADOOP_CONF_DIR' + dest: /home/hduser/.bashrc + state: present + regexp: '^HADOOP_CONF_DIR' line: 'HADOOP_CONF_DIR=$HADOOP_YARN_HOME/conf' -- file: - path: "/{{hadoop_yarn_home}}/conf/slaves" +- file: + path: "/{{hadoop_yarn_home}}/conf/slaves" state: touch -- lineinfile: - dest: "/{{hadoop_yarn_home}}/conf/slaves" - state: present - regexp: "^{{item}}" +- lineinfile: + dest: "/{{hadoop_yarn_home}}/conf/slaves" + state: present + regexp: "^{{item}}" line: "{{item}}" with_items: "{{yarn_slaves}}" -# # Running common tasks in slaves -# - name: Running common tasks in slaves -# include: common.yml -# delegate_to: "{{slave}}" -# with_items: "{{yarn_slaves}}" -# loop_control: -# loop_var: slave - -- name: Synchronizing "{{hadoop_yarn_home}}" with yarn slaves - synchronize: - src: "{{hadoop_yarn_home}}" - dest: "{{hadoop_yarn_home}}" - recursive: yes - delegate_to: "{{ item }}" - loop: "{{yarn_slaves}}" +# Running common tasks in slaves +- name: Running common tasks in slaves + include: common.yml + with_items: "{{yarn_slaves}}" + loop_control: + loop_var: slave - name: Copy truncate_files.sh - copy: - src: truncate_logs.sh + copy: + src: truncate_logs.sh dest: /usr/local/bin mode: 755 - name: Add truncate logs to cron - cron: - name: "Truncate yarn logs" - minute: "0" - job: "/usr/local/bin/truncate_logs.sh" + cron: + name: "Truncate yarn logs" + minute: "0" + job: "/usr/local/bin/truncate_logs.sh" backup: yes diff --git a/ansible/roles/yarn/templates/config.j2 b/ansible/roles/yarn/templates/config.j2 new file mode 100644 index 0000000000..f1b411ed5f --- /dev/null +++ b/ansible/roles/yarn/templates/config.j2 @@ -0,0 +1,3 @@ +Host {{item}} + IdentityFile ~/.ssh/hadoop_rsa + StrictHostKeyChecking no diff --git a/ansible/roles/yarn/templates/hadoop-env.sh b/ansible/roles/yarn/templates/hadoop-env.sh index 5f208be6a0..fd200c373c 100644 --- a/ansible/roles/yarn/templates/hadoop-env.sh +++ b/ansible/roles/yarn/templates/hadoop-env.sh @@ -1 +1,2 @@ -export JAVA_HOME=/opt/jdk1.8.0_121/ +#export JAVA_HOME=/usr/lib/jvm/java-8-oracle +export JAVA_HOME=/opt/jdk1.8.0_121 diff --git a/ansible/roles/yarn/templates/yarn-site.xml b/ansible/roles/yarn/templates/yarn-site.xml index bacb75fa8a..796f6450c3 100644 --- a/ansible/roles/yarn/templates/yarn-site.xml +++ b/ansible/roles/yarn/templates/yarn-site.xml @@ -37,6 +37,14 @@ yarn.nodemanager.log.retain-seconds 3600 + + + yarn.nodemanager.recovery.enabled + true + + + yarn.nodemanager.address + 0.0.0.0:45454 yarn.nodemanager.resource.cpu-vcores