Merge pull request #821 from project-sunbird/test-241_RC1

Fixes from 2.5.0

ansible/es_azure_plugin.yml

@@ -18,7 +18,7 @@
 
     - name: Install plugin
       shell: ES_PATH_CONF=/etc/elasticsearch/"{{ es_instance_name }}" /usr/share/elasticsearch/bin/elasticsearch-plugin install repository-azure
-      when: plugin_output.stdout_lines[0] != "repository-azure"
+      when: (plugin_output.stdout_lines | length == 0) or (plugin_output.stdout_lines[0] != "repository-azure")
 
     - name: add azure storage details in config file
       become: yes

ansible/lp_learning_neo4j_provision.yml

@@ -1,3 +1,8 @@
+- import_playbook: artifacts-download.yml
+  vars:
+    artifact: "{{ neo4j_zip }}"
+    artifact_path: "{{ playbook_dir }}/artifacts/{{ artifact }}"
+
 - hosts: learning-neo4j-cluster
   vars_files:
     - "{{inventory_dir}}/secrets.yml"

ansible/lp_yarn_provision.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: lp-yarn
+- hosts: yarn
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
@@ -17,14 +17,12 @@
         shell: /bin/bash
 
 - name: Install samza job server
-  hosts: "lp-yarn-master"
+  hosts: "yarn-master"
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   become: yes
-  vars_files:
-    - "{{inventory_dir}}/secrets/lp_secrets.yml"
   roles:
-    - java
+    - jdk-1.8.0_121
     - yarn
     - samza-job-server
 
@@ -40,12 +38,10 @@
 #     - include: ./roles/yarn/templates
 
 - name: Install java on all yarn slaves
-  hosts: "lp-yarn-slave"
+  hosts: "yarn-slave"
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   become: yes
-  vars_files:
-    - "{{inventory_dir}}/secrets/lp_secrets.yml"
   remote_user: hduser
   roles:
-    - java
+    - jdk-1.8.0_121

ansible/roles/setup-kafka/defaults/main.yml

@@ -86,6 +86,10 @@ processing_kafka_topics:
   - name: coursebatch.certificate.request
     num_of_partitions: 1
     replication_factor: 1
+  - name: system.command
+    retention_time: 3600
+    num_of_partitions: 1
+    replication_factor: 1 
 
 processing_kafka_overriden_topics:
   - name: telemetry.raw
@@ -161,3 +165,6 @@ processing_kafka_overriden_topics:
   - name: coursebatch.certificate.request
     retention_time: 1209600000
     replication_factor: 1
+  - name: system.command
+    retention_time: 3600
+    replication_factor: 1

ansible/roles/yarn/templates/hadoop-env.sh

@@ -1 +1 @@
-export JAVA_HOME=/usr/lib/jvm/java-8-oracle
+export JAVA_HOME=/opt/jdk1.8.0_121/

pipelines/provision/neo4j/Jenkinsfile

@@ -20,16 +20,6 @@ node() {
                 envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
                 module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-		dir('private'){
-                checkout scm: [$class: 'GitSCM', branches: [[name: private_repo_branch]], extensions: [[$class: 'GitLFSPull'], [$class: 'CloneOption', depth: 1, noTags: true, reference: '', shallow: true]], userRemoteConfigs: [[credentialsId: private_repo_credentials, url: private_repo_url]]]
-		}
-                fileExists = sh(returnStdout: true, script: "find ${currentWs} -name 'neo4j*.tar.gz'")
-                  if (fileExists != "")
-                      sh "cp private/artifacts/neo4j*.tar.gz ansible/artifacts"
-                  else {
-                      println(ANSI_BOLD + ANSI_RED + "Uh Oh! Unable to find neo4j*.tar.gz. Please place this file in your private repo under artifact directory" + ANSI_NORMAL)
-                      error 'Please resolve the errors and rerun..'
-_                 }
                 ansiblePlaybook = "${currentWs}/ansible/lp_learning_neo4j_provision.yml"
                 ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
@@ -53,4 +43,4 @@ _                 }
         slack_notify(currentBuild.result)
         email_notify()
     }
-}
+}