From a8521c5fb26a2f3d708220178eb4287b11e0b400 Mon Sep 17 00:00:00 2001 From: dhuo Date: Fri, 11 Sep 2015 20:24:16 -0700 Subject: [PATCH] New version numbers and updated changelogs for bdutil and gcs/bq connectors. ------------- Created by MOE: http://code.google.com/p/moe-java MOE_MIGRATED_REVID=102899356 --- CHANGES.txt | 23 ++++++++++++++++++++--- bdutil | 2 +- bdutil_env.sh | 4 ++-- hadoop2_env.sh | 4 ++-- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a005fd3..dfe0384 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,9 +1,26 @@ -1.3.2 - 2015-xx-xx - - 1. Added support for setting up a standalone NFS cache server for GCS +1.3.2 - 2015-09-15 + + 1. Updated Spark configurations to make Cloud Bigtable work with Spark. + 2. Added wrappers bigtable-spark-shell and bigtable-spark-submit to use + with bigtable plugin; only installed if bigtable_env.sh is used. + 3. Updated default Hadoop 2 version to 2.7.1. + 4. Added support for Apache Hama. + 5. Added support for setting up a standalone NFS cache server for GCS consistency using standalone_nfs_cache_env.sh, along with configurable GCS_CACHE_MASTER_HOSTNAME to point subsequent clusters at the shared NFS cache server. See standalone_nfs_cache_env.sh for usage. + 6. Added explicit check for ordering of imports spark_env.sh relative + to bigtable_env.sh; Spark must come before Bigtable. + 7. Fixed spelling of "amount" in some documentation. + 8. Fixed directory resolution to bdutil when using symlinks. + 9. Added Dockerfile for bdutil. + 10. Updated default Spark version to 1.5.0; for Spark 1.5.0+, core-site.xml + will also set 'fs.gs.reported.permissions' to 733, otherwise the Hive + 1.2.1 will error out when using SparkSQL. Hadoop MapReduce will print + a harmless warning in this case, but otherwise works fine. Additionally, + Spark auto-restart configuration now contains logic to use correct + syntax for start-slave.sh depending on whether it's Spark 1.4+, and + Spark auto-restarted daemons now correctly run under user 'hadoop'. 1.3.1 - 2015-07-09 diff --git a/bdutil b/bdutil index e138f24..390a5aa 100755 --- a/bdutil +++ b/bdutil @@ -20,7 +20,7 @@ # # Usage: ./bdutil [deploy|delete] [optional: ]... -BDUTIL_VERSION='1.3.2-SNAPSHOT' +BDUTIL_VERSION='1.3.2' # Prints the usage for this script and exits. function print_usage() { diff --git a/bdutil_env.sh b/bdutil_env.sh index 175d23c..15b17a8 100644 --- a/bdutil_env.sh +++ b/bdutil_env.sh @@ -199,10 +199,10 @@ CORES_PER_REDUCE_TASK=1.0 JAVAOPTS='-Xms1024m -Xmx2048m' # Complete URL for downloading the GCS Connector JAR file. -GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.4.2-SNAPSHOT-hadoop1-20150811-194248.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1470885053&Signature=YP67Ay9hggGFZqL4jpKLzlCj9bEQXCckLwcf9%2Bvtzjuequrib11qoVkAT4JTzbURNKZixsBAbox0Y8wjHYyGmo%2Fo90FtJOGt%2BsvD%2FevVeSoppPbekvmrOd1zLNQEFzE4G433v%2Bs5cjx04n%2Fnet6bFy0zp9FjTaqmU0EkdZ0QTUw=' +GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.4.2-hadoop1.jar' # Complete URL for downloading the BigQuery Connector JAR file. -BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.7.2-SNAPSHOT-20150811-202248-hadoop1.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1470886763&Signature=G7b8suYJoLIB1n1G1MuQiwA0WXicLwfNCicXPAQ5es3%2BajzoGp%2BJak%2FmWHrt%2FjXXYBS9veCgEwLLEEJzJxLdlplEXAWJSlOol%2F%2BWsOByJtdUbJ%2Beu20Y3E86y2EhyWtQusXxvIwEhN4%2F9pbFQUmJdIIFSWRUJzUPwMgQvub%2B0Vk=' +BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.7.2-hadoop1.jar' # Complete URL for downloading the configuration script. BDCONFIG='https://storage.googleapis.com/hadoop-tools/bdconfig/bdconfig-0.28.1.tar.gz' diff --git a/hadoop2_env.sh b/hadoop2_env.sh index 424bb9b..6bff058 100644 --- a/hadoop2_env.sh +++ b/hadoop2_env.sh @@ -44,9 +44,9 @@ CORES_PER_REDUCE_TASK=2.0 CORES_PER_APP_MASTER=2.0 # Connector with Hadoop AbstractFileSystem implemenation for YARN -GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.4.2-SNAPSHOT-hadoop2-20150811-194248.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1470885053&Signature=KUIBwGL921BgPiay59Zynz2cBI1Tv45XeXUODPDrnAp0yTksNqeixlHGGUaPfLLxIBEohtkTvgW5oPB8LB0aoQ719ULpiy6SX88W5MlIcl97QDDyUP%2Bz4ZaWXLG6BGY6UNpqkUEFEa4NDg3YIFRsstUr824H5WYAjS1q4sqbdZU=' +GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.4.2-hadoop2.jar' -BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.7.2-SNAPSHOT-20150811-202248-hadoop2.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1470886764&Signature=bYetfZu0t%2Bu9qBNwf59XVZbN3%2Fz%2BnqyyCEAxUAz0rriklZXeq%2F9XaOsMG4wk5TURjP2cuNoS%2Bk7%2FBb4BB0E2sFbr7O6QQA1sOk8zsdDNxDG19cxolSSh98Lx6wAGQcOaqmjwqb2IDwCYMrguWaHDbBC%2FJYGMMGEMBHofR5NXg1U=' +BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.7.2-hadoop2.jar' HDFS_DATA_DIRS_PERM='700'