diff --git a/aws_emr_blog_v3/scripts/emr-tls/create-tls-certs-using-acmpca.sh b/aws_emr_blog_v3/scripts/emr-tls/create-tls-certs-using-acmpca.sh new file mode 100644 index 0000000..07edf38 --- /dev/null +++ b/aws_emr_blog_v3/scripts/emr-tls/create-tls-certs-using-acmpca.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +#================================================================ +# Script to create SSL private keys/certs and upload to AWS Secretes Manager +#================================================================ +#% SYNOPSIS +#+ create-tls-certs.sh args ... +#% +#% DESCRIPTION +#% Uses openssl to create self-signed keys/certs and uploads to AWS Secretes Manager to +#% be used for Ranger Admin server and EMR security configuration +#% Requirements: openssl, aws cli with profile (profile should have IAM permissions to create and delete AWS secrets) +#% +#% ARGUMENTS +#% arg1 Pass the AWS profile to use - +# You can configure this using the documentation below +# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html +#% arg2 AWS_REGION (AWS region where you want to install the secrets) +#% arg3 ACM_PCA_ARN (ACM PCA ARN) +# refer https://docs.aws.amazon.com/privateca/latest/userguide/creating-managing.html to set up acm pca + +#% EXAMPLES +#% create-tls-certs.sh ranger_demo us-east-1 +#% +#================================================================ +#- IMPLEMENTATION +#- version create-tls-certs.sh 2.0 +#- author Varun Bhamidimarri, Stefano SandonĂ  +#- license MIT license +#- +# +#================================================================ +#================================================================ + +[ $# -lt 3 ] && { echo "Usage: $0 AWS_CLI_profile AWS_REGION ACM_PCA_ARN(To setup follow this link: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html)"; exit 1; } + +set -euo pipefail +set -x + +AWS_PROFILE=$1 +AWS_REGION=$2 +acm_pca_auth_arn=$3 +echo $(tr '[:upper:]' '[:lower:]' <<< "$AWS_REGION") +if [[ $(tr '[:upper:]' '[:lower:]' <<< "$AWS_REGION") = "us-east-1" ]]; then + DEFAULT_EC2_REALM='ec2.internal' + echo "AWS region is us-east-1, will use EC2 realm as ec2.internal" +else + DEFAULT_EC2_REALM='compute.internal' + echo "AWS region is NOT us-east-1, will use EC2 realm as compute.internal" +fi +ranger_agents_certs_path="./ranger-agents" +solr_certs_path="./solr-client" +keystore_location="./ranger-plugin-keystore.jks" +keystore_alias=rangerplugin +keystore_password="changeit" +truststore_location="./ranger-plugin-truststore.jks" +ranger_server_certs_path="./ranger-server" +truststore_password="changeit" +truststore_ranger_server_alias="rangeradmin" +secret_mgr_ranger_private_key="emr/rangerGAagentkey" +secret_mgr_ranger_admin_cert="emr/rangerGAservercert" + +certs_subject="/C=US/ST=TX/L=Dallas/O=EMR/OU=EMR/CN=*.$DEFAULT_EC2_REALM" + +generate_certs() { + DIR_EXISTS="false" + if [ -d "$1" ]; then + echo "$1 directory exists, will not recreate certs" + DIR_EXISTS="true" + fi +# rm -rf $1 + if [[ $DIR_EXISTS = "false" ]]; then + rm -rf $1 + mkdir -p $1 + pushd $1 + openssl req -newkey rsa:4096 -keyout privateKey.pem -out certSignRequestforacmpca.csr -days 365 -nodes -subj ${certs_subject} + get_certificate_arn=$(aws acm-pca issue-certificate --certificate-authority-arn $acm_pca_auth_arn --csr fileb://certSignRequestforacmpca.csr --signing-algorithm "SHA256WITHRSA" --validity Value=365,Type="DAYS" --query "CertificateArn" --output=text --profile $AWS_PROFILE --region $AWS_REGION) + sleep 5 + aws acm-pca get-certificate --certificate-authority-arn $acm_pca_auth_arn --certificate-arn $get_certificate_arn --profile $AWS_PROFILE --region $AWS_REGION --query "Certificate" --output=text > publiccertificate.pem + aws acm-pca get-certificate --certificate-authority-arn $acm_pca_auth_arn --certificate-arn $get_certificate_arn --profile $AWS_PROFILE --region $AWS_REGION --query "CertificateChain" --output=text > trustedCertificates.pem + popd + fi +} +rm -rf ${keystore_location} +rm -rf ${truststore_location} +rm -rf ${keystore_location} +generate_certs ranger-server +generate_certs ranger-agents +generate_certs solr-client +generate_certs emr-certs + + +# Delete existing secrets +aws secretsmanager delete-secret --secret-id emr/rangerGAagentkey --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerGAservercert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 + +## Basic wait for delete to be complete +sleep 30 + +#Place the agent private key and public certificate for agenet +cat ${ranger_agents_certs_path}/privateKey.pem ${ranger_agents_certs_path}/publiccertificate.pem > ${ranger_agents_certs_path}/rangerGAagentKeyChain.pem + +aws secretsmanager create-secret --name emr/rangerGAagentkey \ + --description "X509 Ranger Agent Private Key to be used by EMR Security Config" --secret-string file://${ranger_agents_certs_path}/rangerGAagentKeyChain.pem --profile $AWS_PROFILE --region $AWS_REGION + +#Place the server public certificate for agenet +aws secretsmanager create-secret --name emr/rangerGAservercert \ + --description "Ranger Server Cert" --secret-string file://${ranger_server_certs_path}/publiccertificate.pem --profile $AWS_PROFILE --region $AWS_REGION + +## Others that will be used by the Ranger Admin Server +aws secretsmanager delete-secret --secret-id emr/rangerServerPrivateKey --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerServerPublicCert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerServerTrustCert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 + +aws secretsmanager delete-secret --secret-id emr/rangerPluginPrivateKey --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerPluginPublicCert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerPluginTrustCert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 + +aws secretsmanager delete-secret --secret-id emr/rangerSolrPrivateKey --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerSolrPublicCert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 +aws secretsmanager delete-secret --secret-id emr/rangerSolrTrustCert --force-delete-without-recovery --profile $AWS_PROFILE --region $AWS_REGION --cli-read-timeout 10 --cli-connect-timeout 10 + +sleep 30 + +aws secretsmanager create-secret --name emr/rangerServerPrivateKey --description "Ranger Server Private Key" --secret-string file://${ranger_server_certs_path}/privateKey.pem --profile $AWS_PROFILE --region $AWS_REGION +aws secretsmanager create-secret --name emr/rangerServerPublicCert --description "Ranger Server cert chain" --secret-string file://${ranger_server_certs_path}/publiccertificate.pem --profile $AWS_PROFILE --region $AWS_REGION +aws secretsmanager create-secret --name emr/rangerServerTrustCert --description "Ranger Server trust cert" --secret-string file://${ranger_server_certs_path}/trustedCertificates.pem --profile $AWS_PROFILE --region $AWS_REGION + +aws secretsmanager create-secret --name emr/rangerPluginPrivateKey --description "Ranger Plugin Private Key" --secret-string file://${ranger_agents_certs_path}/privateKey.pem --profile $AWS_PROFILE --region $AWS_REGION +aws secretsmanager create-secret --name emr/rangerPluginPublicCert --description "Ranger Plugin Cert" --secret-string file://${ranger_agents_certs_path}/publiccertificate.pem --profile $AWS_PROFILE --region $AWS_REGION +aws secretsmanager create-secret --name emr/rangerPluginTrustCert --description "Ranger trust cert" --secret-string file://${ranger_agents_certs_path}/trustedCertificates.pem --profile $AWS_PROFILE --region $AWS_REGION + +aws secretsmanager create-secret --name emr/rangerSolrPrivateKey --description "Ranger Solr Private Key" --secret-string file://${solr_certs_path}/privateKey.pem --profile $AWS_PROFILE --region $AWS_REGION +aws secretsmanager create-secret --name emr/rangerSolrPublicCert --description "Ranger Solr Cert" --secret-string file://${solr_certs_path}/publiccertificate.pem --profile $AWS_PROFILE --region $AWS_REGION +aws secretsmanager create-secret --name emr/rangerSolrTrustCert --description "Ranger Solr Cert Chain" --secret-string file://${solr_certs_path}/trustedCertificates.pem --profile $AWS_PROFILE --region $AWS_REGION diff --git a/aws_emr_blog_v3/scripts/install-ranger-admin-server-acmpca.sh b/aws_emr_blog_v3/scripts/install-ranger-admin-server-acmpca.sh new file mode 100644 index 0000000..258f13a --- /dev/null +++ b/aws_emr_blog_v3/scripts/install-ranger-admin-server-acmpca.sh @@ -0,0 +1,381 @@ +#!/bin/bash +set -euo pipefail +set -x +#================================================================ +# Script to setup the Apache Ranger Server +#================================================================ +#% SYNOPSIS +#+ install-ranger-admin-server.sh +#% +#% DESCRIPTION +#% Downloads the scripts used in EMR steps +#% +#% EXAMPLES +#% install-ranger-admin-server.sh dc=awsemr,dc=com binduser@awsemr.com 2.0 s3://aws-bigdata-blog/artifacts/aws-blog-emr-ranger 3.0 .rds.amazonaws.com +#% +#================================================================ +#- IMPLEMENTATION +#- version install-ranger-admin-server.sh 1.0 +#- author Varun Bhamidimarri, Stefano SandonĂ  +#- license MIT license +#- +# +#================================================================ +#================================================================ + +sudo yum -y install java-1.8.0 +sudo yum -y remove java-1.7.0-openjdk +sudo yum -y install krb5-workstation krb5-libs krb5-auth-dialog + +export JAVA_HOME=/usr/lib/jvm/jre +# Define variables +hostname=`hostname -I | xargs` +installpath=/usr/lib/ranger + +ldap_ip_address=$1 +ldap_base_dn=$2 +ldap_bind_user_dn=$3 +ldap_bind_password=$4 +ranger_version=$5 +s3bucket=$6 +project_version=${7-'2.0'} +db_host_name=$8 +db_root_password=$9 +default_region=${10-'us-east-1'} +ldap_server_url=ldap://$ldap_ip_address +ranger_service_def_ver=2.0.0 +#ldap_admin_user=${10} +#ldap_domain_dns=${11} +#ldap_admin_password=${12} + +if [ "$ranger_version" == "2.0" ]; then + ranger_download_version=2.1.0-SNAPSHOT + ranger_service_def_ver=2.0.0 +else + ranger_download_version=1.0.1 +fi + +#sudo sed 's/awsemr.com/ec2.internal awsemr.com\nnameserver 10.0.0.2\n/g' + +ranger_s3bucket=$s3bucket/ranger/ranger-$ranger_download_version +ranger_admin_server=ranger-$ranger_download_version-admin +ranger_user_sync=ranger-$ranger_download_version-usersync + +mysql_jar_location=$s3bucket/ranger/ranger-$ranger_download_version/mysql-connector-java-5.1.39.jar +mysql_jar=mysql-connector-java-5.1.39.jar + + +#certs_s3_location=${s3bucket}/${project_version}/emr-tls/ + +certs_path="/tmp/certs" + +#current_hostname=$(hostname -f) +current_hostname=$(TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` && curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-hostname) +sudo hostname $current_hostname + +HTTP_URL=https://localhost:6182 +ranger_agents_certs_path="${certs_path}/ranger-agents-certs" +ranger_server_certs_path="${certs_path}/ranger-server-certs" +solr_certs_path="${certs_path}/solr-client-certs" + +ranger_admin_keystore_alias="rangeradmin" +ranger_admin_keystore_password="changeit" +ranger_admin_keystore_location="/etc/ranger/admin/conf/ranger-admin-keystore.jks" +ranger_admin_truststore_location="$JAVA_HOME/lib/security/cacerts" +ranger_admin_truststore_password="changeit" + + +solr_keystore_location="/etc/solr/conf/solr.jks" +solr_keystore_alias="solr" +solr_keystore_password="changeit" + +truststore_plugins_alias="rangerplugin" +truststore_solr_alias="solrTrust" +truststore_admin_alias="rangeradmin" + +#Download certs +rm -rf ${certs_path} +mkdir -p ${certs_path} +#aws s3 sync ${certs_s3_location} ${certs_path} + +mkdir -p ${ranger_agents_certs_path} +mkdir -p ${ranger_server_certs_path} +mkdir -p ${solr_certs_path} + +#unzip -o ${ranger_agents_certs_path}.zip -d ${ranger_agents_certs_path} +#unzip -o ${ranger_server_certs_path}.zip -d ${ranger_server_certs_path} +#unzip -o ${solr_certs_path}.zip -d ${solr_certs_path} + +## Using Secrets Manager to get the private Key and certs +yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm || true +yum install jq -y +aws secretsmanager get-secret-value --secret-id emr/rangerServerPrivateKey --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${ranger_server_certs_path}/privateKey.pem +aws secretsmanager get-secret-value --secret-id emr/rangerServerPublicCert --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${ranger_server_certs_path}/publiccertificate.pem +aws secretsmanager get-secret-value --secret-id emr/rangerServerTrustCert --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${ranger_server_certs_path}/trustedCertificates.pem + +aws secretsmanager get-secret-value --secret-id emr/rangerPluginPrivateKey --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${ranger_agents_certs_path}/privateKey.pem +aws secretsmanager get-secret-value --secret-id emr/rangerPluginPublicCert --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${ranger_agents_certs_path}/publiccertificate.pem +aws secretsmanager get-secret-value --secret-id emr/rangerPluginTrustCert --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${ranger_agents_certs_path}/trustedCertificates.pem + +aws secretsmanager get-secret-value --secret-id emr/rangerSolrPrivateKey --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${solr_certs_path}/privateKey.pem +aws secretsmanager get-secret-value --secret-id emr/rangerSolrPublicCert --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${solr_certs_path}/publiccertificate.pem +aws secretsmanager get-secret-value --secret-id emr/rangerSolrTrustCert --version-stage AWSCURRENT --region $default_region | jq -r ".SecretString" > ${solr_certs_path}/trustedCertificates.pem + + +sudo mkdir -p /etc/ranger/admin/conf + +#Setup Keystore for RangerAdmin +openssl pkcs12 -export -in ${ranger_server_certs_path}/publiccertificate.pem -inkey ${ranger_server_certs_path}/privateKey.pem -chain -CAfile ${ranger_server_certs_path}/trustedCertificates.pem -name ${ranger_admin_keystore_alias} -out ${ranger_server_certs_path}/keystore.p12 -password pass:${ranger_admin_keystore_password} +keytool -delete -alias ${ranger_admin_keystore_alias} -keystore ${ranger_admin_keystore_location} -storepass ${ranger_admin_keystore_password} -noprompt || true +keytool -importkeystore -deststorepass ${ranger_admin_keystore_password} -destkeystore ${ranger_admin_keystore_location} -srckeystore ${ranger_server_certs_path}/keystore.p12 -srcstoretype PKCS12 -srcstorepass ${ranger_admin_keystore_password} + +#Setup Truststore - add agent cert to Ranger Admin +keytool -delete -alias ${truststore_plugins_alias} -keystore ${ranger_admin_truststore_location} -storepass changeit -noprompt || true +keytool -import -file ${ranger_agents_certs_path}/trustedCertificates.pem -alias ${truststore_plugins_alias} -keystore ${ranger_admin_truststore_location} -storepass ${ranger_admin_truststore_password} -noprompt + +#Setup Truststore - add Solr cert to Ranger Admin +keytool -delete -alias ${truststore_solr_alias} -keystore ${ranger_admin_truststore_location} -storepass changeit -noprompt || true +keytool -import -file ${solr_certs_path}/trustedCertificates.pem -alias ${truststore_solr_alias} -keystore ${ranger_admin_truststore_location} -storepass ${ranger_admin_truststore_password} -noprompt + +#Setup Truststore - add RangerServer cert +keytool -delete -alias ${truststore_admin_alias} -keystore ${ranger_admin_truststore_location} -storepass changeit -noprompt || true +keytool -import -file ${ranger_server_certs_path}/trustedCertificates.pem -alias ${truststore_admin_alias} -keystore ${ranger_admin_truststore_location} -storepass ${ranger_admin_truststore_password} -noprompt + +#Setup Keystore SOLR +mkdir -p /etc/solr/conf +openssl pkcs12 -export -in ${solr_certs_path}/publiccertificate.pem -inkey ${solr_certs_path}/privateKey.pem -chain -CAfile ${solr_certs_path}/trustedCertificates.pem -name ${solr_keystore_alias} -out ${solr_certs_path}/keystore.p12 -password pass:${solr_keystore_password} +keytool -delete -alias ${solr_keystore_alias} -keystore ${solr_keystore_location} -storepass ${solr_keystore_password} -noprompt || true +keytool -importkeystore -deststorepass ${solr_keystore_password} -destkeystore ${solr_keystore_location} -srckeystore ${solr_certs_path}/keystore.p12 -srcstoretype PKCS12 -srcstorepass ${solr_keystore_password} + + +# Setup +yum install -y openldap openldap-clients openldap-servers +# Setup LDAP users +#aws s3 cp $s3bucket/${project_version}/inputdata/load-users-new.ldf . +#aws s3 cp $s3bucket/${project_version}/inputdata/modify-users-new.ldf . +#aws s3 cp $s3bucket/${project_version}/scripts/create-users-using-ldap.sh . +#chmod +x create-users-using-ldap.sh +#./create-users-using-ldap.sh $ldap_ip_address $ldap_admin_user@$ldap_domain_dns $ldap_admin_password $ldap_base_dn || true +#Install mySQL +yum -y install mysql-server || true +# New Amazon Linux AMI's use the MariaDB yum package - either mysql or maria DB is required +yum -y install mariadb-server mariadb-libs mariadb || true +service mariadb start || true +chkconfig mariadb on || true +mysqladmin -u root password rangeradmin || true +rm -rf $installpath +mkdir -p $installpath/hadoop +cd $installpath +aws s3 cp $ranger_s3bucket/$ranger_admin_server.tar.gz . +aws s3 cp $ranger_s3bucket/$ranger_user_sync.tar.gz . +aws s3 cp $mysql_jar_location . +aws s3 cp $ranger_s3bucket/solr_for_audit_setup.tar.gz . +#Update ranger admin install.properties +mkdir $ranger_admin_server +tar -xvf $ranger_admin_server.tar.gz -C $ranger_admin_server --strip-components=1 + +cd $ranger_admin_server + +sudo sed -i "s|SQL_CONNECTOR_JAR=.*|SQL_CONNECTOR_JAR=$installpath/$mysql_jar|g" install.properties + +DB_ROOT_USERNAME="root" + +RDS_RANGER_SCHEMA_DBNAME="rangerdb" +RDS_RANGER_SCHEMA_DBUSER="rangeradmin" +RDS_RANGER_SCHEMA_DBPASSWORD="rangeradmin" + +MYSQL="/usr/bin/mysql" + +_generateSQLGrantsAndCreateUser() +{ + touch ~/generate_grants.sql + HOSTNAMEI=`hostname -I` + HOSTNAMEI=`echo ${HOSTNAMEI}` + cat >~/generate_grants.sql <localjceks://file//usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/conf/.jceks/rangeradmin.jceks|g" /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/conf/ranger-admin-default-site.xml + +#Update ranger usersync install.properties +cd $installpath +mkdir $ranger_user_sync +tar -xvf $ranger_user_sync.tar.gz -C $ranger_user_sync --strip-components=1 +cp ./$ranger_admin_server/ews/webapp/WEB-INF/lib/jackson-* ./$ranger_user_sync/lib/ +chown ranger:ranger ./$ranger_user_sync/lib/* +chmod 755 ./$ranger_user_sync/lib/* + +cd $ranger_user_sync + + +#sudo sed -i "s|POLICY_MGR_URL =.*|POLICY_MGR_URL=http://$hostname:6080|g" install.properties + +sudo sed -i "s|POLICY_MGR_URL =.*|POLICY_MGR_URL=https://$current_hostname:6182|g" install.properties +sudo sed -i "s|POLICY_MGR_URL=.*|POLICY_MGR_URL=https://$current_hostname:6182|g" install.properties +sudo sed -i "s|SYNC_SOURCE =.*|SYNC_SOURCE=ldap|g" install.properties +sudo sed -i "s|SYNC_LDAP_URL =.*|SYNC_LDAP_URL=$ldap_server_url|g" install.properties +sudo sed -i "s|SYNC_LDAP_BIND_DN =.*|SYNC_LDAP_BIND_DN=$ldap_bind_user_dn|g" install.properties +sudo sed -i "s|SYNC_LDAP_BIND_PASSWORD =.*|SYNC_LDAP_BIND_PASSWORD=$ldap_bind_password|g" install.properties + + +sudo sed -i "s|SYNC_LDAP_SEARCH_BASE =.*|SYNC_LDAP_SEARCH_BASE=$ldap_base_dn|g" install.properties +sudo sed -i "s|SYNC_LDAP_USER_SEARCH_BASE =.*|SYNC_LDAP_USER_SEARCH_BASE=$ldap_base_dn|g" install.properties +sudo sed -i "s|SYNC_LDAP_USER_SEARCH_FILTER =.*|SYNC_LDAP_USER_SEARCH_FILTER=sAMAccountName=*|g" install.properties +sudo sed -i "s|SYNC_LDAP_USER_NAME_ATTRIBUTE =.*|SYNC_LDAP_USER_NAME_ATTRIBUTE=sAMAccountName|g" install.properties +sudo sed -i "s|SYNC_INTERVAL =.*|SYNC_INTERVAL=2|g" install.properties +# SSL conf +sudo sed -i "s|AUTH_SSL_TRUSTSTORE_FILE=.*|AUTH_SSL_TRUSTSTORE_FILE=$ranger_admin_truststore_location|g" install.properties +sudo sed -i "s|AUTH_SSL_TRUSTSTORE_PASSWORD=.*|AUTH_SSL_TRUSTSTORE_PASSWORD=$ranger_admin_truststore_password|g" install.properties + +sudo cp /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/lib/commons-configuration* /usr/lib/ranger/$ranger_user_sync/lib/ + +chmod +x setup.sh +./setup.sh + +#Download the install solr for ranger +cd $installpath +mkdir solr_for_audit_setup +tar -xvf solr_for_audit_setup.tar.gz -C solr_for_audit_setup --strip-components=1 +cd solr_for_audit_setup + +solr_standalone_conf_script="/usr/lib/ranger/solr_for_audit_setup/solr_standalone/scripts/solr.in.sh.j2" + +sudo sed -i "s|SOLR_HOST_URL=.*|SOLR_HOST_URL=https://$current_hostname:8984|g" install.properties +sudo sed -i "s|SOLR_RANGER_PORT=.*|SOLR_RANGER_PORT=8984|g" install.properties + +sudo sed -i "s|.*SOLR_SSL_KEY_STORE=.*|SOLR_SSL_KEY_STORE=${solr_keystore_location}|g" ${solr_standalone_conf_script} +sudo sed -i "s|.*SOLR_SSL_KEY_STORE_PASSWORD=.*|SOLR_SSL_KEY_STORE_PASSWORD=${solr_keystore_password}|g" ${solr_standalone_conf_script} +sudo sed -i "s|.*SOLR_SSL_TRUST_STORE=.*|SOLR_SSL_TRUST_STORE=$JAVA_HOME/lib/security/cacerts|g" ${solr_standalone_conf_script} +sudo sed -i "s|.*SOLR_SSL_TRUST_STORE_PASSWORD=.*|SOLR_SSL_TRUST_STORE_PASSWORD=changeit|g" ${solr_standalone_conf_script} +sudo sed -i "s|.*SOLR_SSL_NEED_CLIENT_AUTH=.*|SOLR_SSL_NEED_CLIENT_AUTH=false|g" ${solr_standalone_conf_script} +sudo sed -i "s|.*SOLR_SSL_WANT_CLIENT_AUTH=.*|SOLR_SSL_WANT_CLIENT_AUTH=false|g" ${solr_standalone_conf_script} + + + +#sudo sed -i "s|SOLR_HOST_URL=.*|SOLR_HOST_URL=http://$hostname:8983|g" install.properties +#sudo sed -i "s|SOLR_RANGER_PORT=.*|SOLR_RANGER_PORT=8983|g" install.properties +sudo sed -i "s|SOLR_MAX_MEM=.*|SOLR_MAX_MEM=4g|g" install.properties +sed -i 's/+90DAYS/+2DAYS/g' conf/solrconfig.xml +chmod +x setup.sh +./setup.sh + +sudo mkdir -p /usr/lib/ranger/logs/admin/ +sudo ln -sfn /usr/lib/ranger/$ranger_admin_server/ews/logs /usr/lib/ranger/logs/admin/logs + +#Start Ranger Admin +sudo echo "log4j.appender.xa_log_policy_appender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.xa_log_policy_appender.file=\${logdir}/ranger_admin_policy_updates.log +log4j.appender.xa_log_policy_appender.datePattern='.'yyyy-MM-dd +log4j.appender.xa_log_policy_appender.append=true +log4j.appender.xa_log_policy_appender.layout=org.apache.log4j.PatternLayout +log4j.appender.xa_log_policy_appender.layout.ConversionPattern=%d [%t] %-5p %C{6} (%F:%L) - %m%n + +log4j.category.org.apache.ranger.rest.ServiceREST=debug,xa_log_policy_appender +log4j.additivity.org.apache.ranger.rest.ServiceREST=false" >> /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/log4j.properties +sudo ln -s /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/hive/ranger-hive-plugin-$ranger_download_version* /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/lib/ +sudo ln -s /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/hdfs/ranger-hdfs-plugin-$ranger_download_version* /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/lib/ + +# Setup the Spark Ranger plugin definition +sudo rm -rf /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/amazon-emr-spark +sudo mkdir -p /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/amazon-emr-spark +sudo wget -O /tmp/ranger-spark-plugin-2.x.jar https://s3.amazonaws.com/elasticmapreduce/ranger/service-definitions/version-2.0/ranger-spark-plugin-2.x.jar +sudo mv /tmp/ranger-spark-plugin-2.x.jar /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/amazon-emr-spark/ + +# Setup the EMRFS Ranger plugin definition +sudo rm -rf /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/amazon-emr-emrfs +sudo mkdir -p /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/amazon-emr-emrfs +sudo wget -O /tmp/ranger-emrfs-s3-plugin-2.x.jar https://s3.amazonaws.com/elasticmapreduce/ranger/service-definitions/version-2.0/ranger-emr-emrfs-plugin-2.x.jar +sudo mv /tmp/ranger-emrfs-s3-plugin-2.x.jar /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/amazon-emr-emrfs/ + + +#CHECKTHIS - wrong path +sudo cp /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/classes/ranger-plugins/hive/* /usr/lib/ranger/$ranger_admin_server/ews/webapp/WEB-INF/lib/ || true + +#Setup proper owner for keytabs locations +sudo chown solr:solr -R /etc/solr +sudo chown ranger:ranger -R /etc/ranger + +#cleanup +rm -rf ${certs_path} + +sudo /usr/bin/ranger-admin stop || true +sudo /usr/bin/ranger-admin start#!/bin/bash \ No newline at end of file