From 26eb5a5883f28338c7bfb23286184638f0e9d2c5 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Mon, 27 May 2024 10:41:33 +0530 Subject: [PATCH 1/3] Bump hive version to 4.0.0 as 2.x.x is EOL --- .circleci/integration-tests/Dockerfile.astro_cloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/integration-tests/Dockerfile.astro_cloud b/.circleci/integration-tests/Dockerfile.astro_cloud index 902700f69..c08996a35 100644 --- a/.circleci/integration-tests/Dockerfile.astro_cloud +++ b/.circleci/integration-tests/Dockerfile.astro_cloud @@ -38,7 +38,7 @@ RUN apt-get update -y \ # Set Hive and Hadoop versions. -ENV HIVE_LIBRARY_VERSION=hive-2.3.9 +ENV HIVE_LIBRARY_VERSION=hive-4.0.0 ENV HADOOP_LIBRARY_VERSION=hadoop-2.10.1 # install AWS CLI From a943b1dc4f559473403e9df5c4efe99ba234f388 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Mon, 27 May 2024 13:54:31 +0530 Subject: [PATCH 2/3] update hive and emr version --- .circleci/integration-tests/Dockerfile.astro_cloud | 2 +- astronomer/providers/apache/hive/example_dags/example_hive.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/integration-tests/Dockerfile.astro_cloud b/.circleci/integration-tests/Dockerfile.astro_cloud index c08996a35..ac2c3cebf 100644 --- a/.circleci/integration-tests/Dockerfile.astro_cloud +++ b/.circleci/integration-tests/Dockerfile.astro_cloud @@ -38,7 +38,7 @@ RUN apt-get update -y \ # Set Hive and Hadoop versions. -ENV HIVE_LIBRARY_VERSION=hive-4.0.0 +ENV HIVE_LIBRARY_VERSION=hive-3.1.2 ENV HADOOP_LIBRARY_VERSION=hadoop-2.10.1 # install AWS CLI diff --git a/astronomer/providers/apache/hive/example_dags/example_hive.py b/astronomer/providers/apache/hive/example_dags/example_hive.py index 7cd2deecf..d759e8d38 100644 --- a/astronomer/providers/apache/hive/example_dags/example_hive.py +++ b/astronomer/providers/apache/hive/example_dags/example_hive.py @@ -55,7 +55,7 @@ # match the hive and hadoop versions same as specified in the integration tests `Dockefile`. JOB_FLOW_OVERRIDES = { "Name": HIVE_CLUSTER, - "ReleaseLabel": "emr-5.34.0", + "ReleaseLabel": "emr-6.6.0", "Applications": [ {"Name": "Spark"}, { From 8c4e797027aa4593fe53f3bdac2e25271a09a282 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Mon, 27 May 2024 15:08:56 +0530 Subject: [PATCH 3/3] updating hive version to 3.1.3 --- .circleci/integration-tests/Dockerfile.astro_cloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/integration-tests/Dockerfile.astro_cloud b/.circleci/integration-tests/Dockerfile.astro_cloud index ac2c3cebf..18462a037 100644 --- a/.circleci/integration-tests/Dockerfile.astro_cloud +++ b/.circleci/integration-tests/Dockerfile.astro_cloud @@ -38,7 +38,7 @@ RUN apt-get update -y \ # Set Hive and Hadoop versions. -ENV HIVE_LIBRARY_VERSION=hive-3.1.2 +ENV HIVE_LIBRARY_VERSION=hive-3.1.3 ENV HADOOP_LIBRARY_VERSION=hadoop-2.10.1 # install AWS CLI