Skip to content

Commit 2d706fa

Browse files
cgiraldocgiraldo
committed
Run hive as "hive" user. Set Hadoop-config through configMap.
1 parent 7ff1782 commit 2d706fa

11 files changed

+148
-30
lines changed

‎charts/hive/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ sources:
1313
- https://github.com/apache/hive
1414
- https://github.com/gradiant/charts
1515
- https://github.com/big-data-europe/docker-hive
16-
version: 0.1.1
16+
version: 0.1.2

‎charts/hive/resources/config/core-site.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<configuration>
4+
{{- range $key, $value := index .Values.conf "coreSite" }}
5+
<property><name>{{ $key }}</name><value>{{ $value }}</value></property>
6+
{{- end }}
7+
</configuration>

‎charts/hive/resources/config/hdfs-site.xml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<configuration>
4+
{{- range $key, $value := index .Values.conf "hdfsSite" }}
5+
<property><name>{{ $key }}</name><value>{{ $value }}</value></property>
6+
{{- end }}
7+
8+
</configuration>

‎charts/hive/resources/config/hive-site.xml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<configuration>
4+
{{- if not (index .Values.conf "hiveSite" "hive.metastore.uris") }}
5+
<property>
6+
<name>hive.metastore.uris</name>
7+
<value>{{- printf "thrift://%s-metastore" .Release.Name }}:9083</value>
8+
</property>
9+
{{- end }}
10+
{{- if index .Values.conf "hiveSite" }}
11+
{{- range $key, $value := index .Values.conf "hiveSite" }}
12+
<property><name>{{ $key }}</name><value>{{ $value }}</value></property>
13+
{{- end }}
14+
{{- end }}
15+
</configuration>

‎charts/hive/resources/config/mapred-site.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<configuration>
4+
{{- range $key, $value := index .Values.conf "mapredSite" }}
5+
<property><name>{{ $key }}</name><value>{{ $value }}</value></property>
6+
{{- end }}
7+
</configuration>

‎charts/hive/resources/config/yarn-site.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<configuration>
4+
{{- range $key, $value := index .Values.conf "yarnSite" }}
5+
<property><name>{{ $key }}</name><value>{{ $value }}</value></property>
6+
{{- end }}
7+
</configuration>

‎charts/hive/resources/startup.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
: ${HADOOP_PREFIX:=/usr/local/hadoop}
4+
5+
. $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
6+
7+
# Directory to find config artifacts
8+
CONFIG_DIR="/tmp/hadoop-config"
9+
10+
set -x
11+
12+
# Copy config files from volume mount
13+
for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml; do
14+
if [[ -e ${CONFIG_DIR}/$f ]]; then
15+
cp ${CONFIG_DIR}/$f $HADOOP_HOME/etc/hadoop/$f
16+
else
17+
echo "ERROR: Could not find $f in $CONFIG_DIR"
18+
exit 1
19+
fi
20+
done
21+
22+
# Note. This script set hive paths in hdfs with user hive and ensures hiveServer is runAsUser hive
23+
24+
HADOOP_USER_NAME={{ .Values.conf.hdfsAdminUser }} hdfs dfs -mkdir /tmp
25+
HADOOP_USER_NAME={{ .Values.conf.hdfsAdminUser }} hdfs dfs -mkdir -p /user/hive/warehouse
26+
HADOOP_USER_NAME={{ .Values.conf.hdfsAdminUser }} hdfs dfs -chmod g+w /tmp
27+
HADOOP_USER_NAME={{ .Values.conf.hdfsAdminUser }} hdfs dfs -chmod g+w /user/hive/warehouse
28+
HADOOP_USER_NAME={{ .Values.conf.hdfsAdminUser }} hdfs dfs -chown hive:hive /tmp
29+
HADOOP_USER_NAME={{ .Values.conf.hdfsAdminUser }} hdfs dfs -chown hive:hive /user/hive/warehouse
30+
31+
32+
if id -u hive ; then
33+
echo "hive user exists";
34+
else
35+
echo "Creating hive user";
36+
groupadd -g 500 -r hive && \
37+
useradd --comment "Hive user" -u 500 --shell /bin/bash -M -r -g hive hive
38+
fi
39+
40+
if [[ whoami != hive ]]
41+
then
42+
echo "Switching to hive user";
43+
su hive -c "cd $HIVE_HOME/bin; ./hiveserver2 --hiveconf hive.server2.enable.doAs=false"
44+
else
45+
cd $HIVE_HOME/bin; ./hiveserver2 --hiveconf hive.server2.enable.doAs=false
46+
fi

‎charts/hive/templates/hadoop-configmap.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{{- if .Values.conf.hadoopSite }}
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: {{ include "hive.fullname" . }}-hadoop
6+
labels:
7+
app.kubernetes.io/name: {{ include "hive.name" . }}
8+
{{- include "hive.labels" . | nindent 4 }}
9+
data:
10+
# TODO check if we need to include all this files for hive
11+
core-site.xml: |-
12+
{{ tpl (.Files.Get "resources/config/core-site.xml") . | indent 4 }}
13+
hdfs-site.xml: |-
14+
{{ tpl (.Files.Get "resources/config/hdfs-site.xml") . | indent 4 }}
15+
mapred-site.xml: |-
16+
{{ tpl (.Files.Get "resources/config/mapred-site.xml") . | indent 4 }}
17+
yarn-site.xml: |-
18+
{{ tpl (.Files.Get "resources/config/yarn-site.xml") . | indent 4 }}
19+
{{- end }}

‎charts/hive/templates/hive-configmap.yaml

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,7 @@ metadata:
66
app.kubernetes.io/name: {{ include "hive.name" . }}
77
{{- include "hive.labels" . | nindent 4 }}
88
data:
9+
startup.sh: |-
10+
{{ tpl (.Files.Get "resources/startup.sh") . | indent 4 }}
911
hive-site.xml: |
10-
<?xml version="1.0"?>
11-
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
12-
<configuration>
13-
{{- if not (index .Values.conf "hiveSite" "hive.metastore.uris") }}
14-
<property>
15-
<name>hive.metastore.uris</name>
16-
<value>{{- printf "thrift://%s-metastore" .Release.Name }}:9083</value>
17-
</property>
18-
{{- end }}
19-
{{- if index .Values.conf "hiveSite" }}
20-
{{- range $key, $value := index .Values.conf "hiveSite" }}
21-
<property><name>{{ $key }}</name><value>{{ $value }}</value></property>
22-
{{- end }}
23-
{{- end }}
24-
</configuration>
12+
{{ tpl (.Files.Get "resources/config/hive-site.xml") . | indent 4 }}

‎charts/hive/templates/hive-statefulset.yaml

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,11 @@ spec:
2525
- name: server
2626
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
2727
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
28+
command:
29+
- /bin/bash
30+
- /opt/hive/conf/startup.sh
2831
resources:
2932
{{ toYaml .Values.resources | indent 10 }}
30-
env:
31-
# TODO change this to use hadoop Configmap instead of env variables
32-
- name: CORE_CONF_fs_defaultFS
33-
{{- if not (index .Values.conf "coreSite" "fs.defaultFS") }}
34-
value: hdfs://{{.Release.Name}}-hdfs-namenode:8020
35-
{{- else }}
36-
value: {{ index .Values.conf "coreSite" "fs.defaultFS" }}
37-
{{- end }}
3833
readinessProbe:
3934
httpGet:
4035
path: /
@@ -50,7 +45,20 @@ spec:
5045
volumeMounts:
5146
- name: hive-config
5247
mountPath: /opt/hive/conf
48+
- name: hadoop-config
49+
mountPath: /tmp/hadoop-config
5350
volumes:
51+
- name: hadoop-config
52+
configMap:
53+
{{- if .Values.conf.hadoopSite }}
54+
name: {{ include "hive.fullname" . }}-hadoop
55+
{{- else }}
56+
{{- if .Values.conf.hadoopConfigMap }}
57+
name: {{ .Values.conf.hadoopConfigMap }}
58+
{{- else }}
59+
name: {{ .Release.Name }}-hdfs-hadoop
60+
{{- end }}
61+
{{- end }}
5462
- name: hive-config
5563
configMap:
5664
name: {{ include "hive.fullname" . }}

‎charts/hive/values.yaml

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,28 @@ image:
99

1010
resources: {}
1111

12-
conf:
13-
coreSite:
14-
fs.defaultFS: # default is hdfs://{{.Release.Name}}-hdfs-namenode:8020
15-
hiveSite:
16-
hive.metastore.uris: # default is "thrift://{{.Release.Name}}-metastore:9083"
17-
12+
# Also deploy hive-metastore requirement
1813
metastore:
1914
enabled: true
2015

16+
# Also deploy hdfs requirement
2117
hdfs:
2218
enabled: true
2319

20+
conf:
21+
hiveSite:
22+
# if not set, default hive.metastore.uris is default uri
23+
# from metastore requirement: "thrift://{{.Release.Name}}-metastore:9083"
24+
hive.metastore.uris:
25+
hdfsAdminUser: hdfs
26+
# if not set, default is configMap from hdfs requirement {{.Release.Name}}-hdfs-hadoop
27+
hadoopConfigMap:
28+
# to manually provide hadoop config attributes instead of hadoopConfigMap.
29+
# hadoopSite:
30+
# coreSite:
31+
# fs.defaultFS: hdfs://hdfs-cluster:8020
32+
# hdfsSite:
33+
# ...
34+
35+
36+

0 commit comments

Comments
 (0)