W3M2 : feat modify, verification, mapreduce script

missions/W3/M2/Dockerfile.datanode

@@ -53,6 +53,9 @@ COPY ./config/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
 COPY ./config/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
 COPY ./config/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
 
+# chown Hadoop configuration files
+RUN sudo chown -R hadoop:hadoop $HADOOP_HOME
+
 # Configure JAVA_HOME in Hadoop environment
 RUN echo "export JAVA_HOME=$JAVA_HOME" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
 

missions/W3/M2/Dockerfile.namenode

@@ -53,6 +53,9 @@ COPY ./config/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
 COPY ./config/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
 COPY ./config/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
 
+# chown Hadoop configuration files
+RUN sudo chown -R hadoop:hadoop $HADOOP_HOME
+
 # Configure JAVA_HOME in Hadoop environment
 RUN echo "export JAVA_HOME=$JAVA_HOME" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
 

missions/W3/M2/Dockerfile.nodemanager

@@ -53,6 +53,9 @@ COPY ./config/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
 COPY ./config/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
 COPY ./config/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
 
+# chown Hadoop configuration files
+RUN sudo chown -R hadoop:hadoop $HADOOP_HOME
+
 # Configure JAVA_HOME in Hadoop environment
 RUN echo "export JAVA_HOME=$JAVA_HOME" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
 

missions/W3/M2/Dockerfile.resourcemanager

@@ -53,6 +53,9 @@ COPY ./config/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml
 COPY ./config/mapred-site.xml $HADOOP_HOME/etc/hadoop/mapred-site.xml
 COPY ./config/yarn-site.xml $HADOOP_HOME/etc/hadoop/yarn-site.xml
 
+# chown Hadoop configuration files
+RUN sudo chown -R hadoop:hadoop $HADOOP_HOME
+
 # Configure JAVA_HOME in Hadoop environment
 RUN echo "export JAVA_HOME=$JAVA_HOME" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh
 

missions/W3/M2/README.md

@@ -1 +1,69 @@
-# INIT 
+# INIT 
+
+## Build initial hadoop cluster
+* step 1 : run "build_and_run_hadoop_services.sh"
+instruction | ./build_and_run_hadoop_services.sh
+
+## Change Configuration
+* step 1 : Modify the configuration files in "/change-config".
+* step 2 : Add the directories that need to be included for configuration changes at the bottom of the "make_dir.sh" file.
+<예시> 
+''' xml
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>file:///hadoop/dfs/name</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>file:///hadoop/dfs/data</value>
+'''
+
+When changing as above, /hadoop/dfs/name and /hadoop/dfs/data paths must be added inside the container.
+
+* step 3 : 변경 사항을 적용할 컨테이너를 apply_all.sh에 추가합니다
+./configuration_modify.sh $HADOOP_HOME $CONTAINER_NAME $ROLE 오 같은 명령행을 추가합니다.
+
+* step 4 : run "apply_all.sh"
+instruction | ./apply_all.sh
+변경사항이 적용됩니다.
+
+
+## Verification
+* step 1 : run "build-verify-scrips.py"
+instruction | python3 run build-verify-scrips.py
+it creates four .sh files for verify changed configuration
+"verify_core-site_conf.sh", "verify_hdfs-site_conf.sh", "verify_mapred-site_conf.sh", "verify_yarn-site_conf.sh" 
+
+The above four scripts are required to run configuration_verify.sh
+
+* step 2 : run "configuration_verify.sh"
+instruction : ./configuration_verify.sh <HADOOP_HOME> <CONTAINER_NAME>
+
+## mapreduce
+* just run test_mapreduce.sh
+instruction : ./test_mapreduce.sh <HADOOP_HOME> <CONTAINER_NAME>
+
+* if you want to test by using another mapreduce process
+* chagene input.txt, wordcount.sh
+
+
+## TEST
+* if you only want to test, not customise then, just follow instructions below
+instruction | ./build_and_run_hadoop_services.sh
+instruction | ./apply_all.sh
+instruction | python3 run build-verify-scrips.py
+instruction : ./configuration_verify.sh usr/local/hadoop namenode
+instruction : ./test_mapreduce.sh usr/local/hadoop namenode
+
+
+## Troble Shooting
+if you change dfs.datanode.data.dir property, then there is a possibility that you can get "java.io.IOException: Incompatible clusterIDs in /hadoop/dfs/data: namenode clusterID = CID-8ec62c1c-7b9a-413b-afa2-05dd41fc8f94; datanode clusterID = CID-79c89a70-9b81-4808-a954-d6d4d8c98c02"
+
+This is because the directory with the changed settings already exists. you should remove the directory and run script again
+
+''' xml
+<property>
+    <name>dfs.datanode.data.dir</name>
+    <value>file:///usr/local/hadoop/data/datanode</value>
+</property>
+'''

missions/W3/M2/change-config/apply_all.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# configuration_modify.sh $HADOOP_HOME $CONTAINER_NAME $ROLE
+./configuration_modify.sh usr/local/hadoop namenode namenode
+./configuration_modify.sh usr/local/hadoop resourcemanager resourcemanager
+./configuration_modify.sh usr/local/hadoop datanode1 datanode
+./configuration_modify.sh usr/local/hadoop datanode2 datanode
+./configuration_modify.sh usr/local/hadoop nodemanager1 nodemanager
+./configuration_modify.sh usr/local/hadoop nodemanager2 nodemanager

missions/W3/M2/change-config/configuration_modify.sh

@@ -0,0 +1,192 @@
+#!/bin/bash
+
+# 인자를 3개 받는다. 이때 첫 번째 인자는 HADOOP_HOME, 두 번째 인자는 CONTAINER_NAME, 세번째 인자는 하둡 서비스 종류이다.
+# SERVICE_TYPE = {namenode, datanode, resourcemanager, nodemanager}
+if [ $# -ne 3 ]; then
+  echo "Usage: $0 <HADOOP_HOME> <CONTAINER_NAME> <SERVICE_TYPE>"
+  exit 1
+fi
+
+# make_dir.sh 실행 인자는 CONTAINER_NAME
+./make_dir.sh $2
+
+# 환경 변수 설정
+HADOOP_HOME=$1
+CONTAINER_NAME=$2
+SERVICE_TYPE=$3
+
+# 올바른 서비스 타입인지 확인
+if [[ ! "namenode datanode resourcemanager nodemanager" =~ (^|[[:space:]])$SERVICE_TYPE($|[[:space:]]) ]]; then
+  echo "Invalid service type. Please specify one of the following: namenode, datanode, resourcemanager, nodemanager."
+  exit 1
+fi
+
+# 함수 정의: 디렉토리 생성
+create_directory() {
+  local dir=$1
+  echo "Creating directory $dir..."
+  docker exec -it $CONTAINER_NAME bash -c "
+    if [ ! -d $dir ]; then
+      mkdir -p $dir
+    fi"
+  if [ $? -ne 0 ]; then
+    echo "Failed to create directory $dir inside the container."
+    exit 1
+  fi
+}
+
+# 함수 정의: 파일 복사
+copy_file() {
+  local src_file=$1
+  local dest_dir=$2
+  echo "Copying $src_file..."
+  docker cp $src_file $CONTAINER_NAME:$dest_dir/
+  if [ $? -ne 0 ]; then
+    echo "Failed to copy $src_file."
+    exit 1
+  fi
+}
+
+# 함수 정의: 파일 백업
+backup_file() {
+  local file=$1
+  local backup_dir=$2
+  echo "Backing up $file..."
+  docker exec -it $CONTAINER_NAME bash -c "
+    timestamp=$timestamp
+    mkdir -p $backup_dir &&
+    cp $file $backup_dir &&
+    echo 'Configuration file[$(basename $file)] have been backed up to $backup_dir.'
+    "
+  if [ $? -ne 0 ]; then
+    echo "Failed to back up configuration file[$(basename $file)] inside the container."
+    exit 1
+  fi
+}
+
+# 함수 정의: 파일 업데이트
+update_file() {
+  local src_file=$1
+  local dest_file=$2
+  echo "Updating $dest_file..."
+  docker exec -it $CONTAINER_NAME bash -c "
+    cp $src_file $dest_file &&
+    rm -rf $src_file &&
+    chmod +x $dest_file &&
+    echo 'Configuration file[$(basename $dest_file)] have been updated and temporary files removed.'"
+  if [ $? -ne 0 ]; then
+    echo "Failed to update configuration file[$(basename $dest_file)] inside the container."
+    exit 1
+  fi
+}
+
+# 함수 정의: namenode 서비스 재시작 (HDFS 서비스)
+restart_namenode_service() {
+  local service=$1
+  local name_dir_origin=$2
+  local name_dir_changed=$3
+  echo "Restarting Hadoop $service service..."
+
+  # Determine whether to format or not
+  if [ "$name_dir_origin" != "$name_dir_changed" ]; then
+    echo "Formatting NameNode directory..."
+    docker exec -it $CONTAINER_NAME bash -c "
+    $HADOOP_HOME/bin/hdfs --daemon stop $service &&
+    $HADOOP_HOME/bin/hdfs namenode -format &&
+    nohup $HADOOP_HOME/bin/hdfs --daemon start $service > /dev/null
+    "
+  else
+    echo "NameNode directory has not changed. Restarting Hadoop $service service..."
+    docker exec -it $CONTAINER_NAME bash -c "
+    $HADOOP_HOME/bin/hdfs --daemon stop $service &&
+    nohup $HADOOP_HOME/bin/hdfs --daemon start $service > /dev/null
+    "
+  fi
+
+  if [ $? -ne 0 ]; then
+    echo "Failed to restart Hadoop $service service."
+    exit 1
+  fi
+  echo "Hadoop $service service has been restarted successfully."
+}
+
+# 함수 정의: datanode 서비스 재시작 (HDFS 서비스)
+restart_datanode_service() {
+  local service=$1
+  echo "Restarting Hadoop $service service..."
+  docker exec -it $CONTAINER_NAME bash -c "
+  $HADOOP_HOME/bin/hdfs --daemon stop $service &&
+  nohup $HADOOP_HOME/bin/hdfs --daemon start $service > /dev/null
+  "
+  if [ $? -ne 0 ]; then
+    echo "Failed to restart Hadoop $service service."
+    exit 1
+  fi
+  echo "Hadoop $service service has been restarted successfully."
+}
+
+# 함수 정의: YARN 서비스 재시작
+restart_yarn_service() {
+  local service=$1
+  echo "Restarting Hadoop $service service..."
+  docker exec -it $CONTAINER_NAME bash -c "
+    $HADOOP_HOME/bin/yarn --daemon stop $service &&
+    nohup $HADOOP_HOME/bin/yarn --daemon start $service > /dev/null
+    "
+  if [ $? -ne 0 ]; then
+    echo "Failed to restart Hadoop $service service."
+    exit 1
+  fi
+  echo "Hadoop $service service has been restarted successfully."
+}
+
+# 타임스탬프 생성
+timestamp=$(date +%s)
+
+# 디렉토리 생성 및 파일 복사
+create_directory "$HADOOP_HOME/etc/hadoop/tmp"
+copy_file "core-site.xml" "$HADOOP_HOME/etc/hadoop/tmp"
+copy_file "hdfs-site.xml" "$HADOOP_HOME/etc/hadoop/tmp"
+copy_file "mapred-site.xml" "$HADOOP_HOME/etc/hadoop/tmp"
+copy_file "yarn-site.xml" "$HADOOP_HOME/etc/hadoop/tmp"
+
+create_directory "$HADOOP_HOME/etc/hadoop/backup/$timestamp"
+
+# 파일 백업
+backup_file "$HADOOP_HOME/etc/hadoop/core-site.xml" "$HADOOP_HOME/etc/hadoop/backup/$timestamp"
+backup_file "$HADOOP_HOME/etc/hadoop/hdfs-site.xml" "$HADOOP_HOME/etc/hadoop/backup/$timestamp"
+backup_file "$HADOOP_HOME/etc/hadoop/mapred-site.xml" "$HADOOP_HOME/etc/hadoop/backup/$timestamp"
+backup_file "$HADOOP_HOME/etc/hadoop/yarn-site.xml" "$HADOOP_HOME/etc/hadoop/backup/$timestamp"
+
+# Get the NameNode directory
+NAME_DIR_ORIGIN=$(docker exec -it $CONTAINER_NAME bash -c "$HADOOP_HOME/bin/hdfs getconf -confKey dfs.namenode.name.dir 2>&1" | awk '/file:\/\// {print $1}' | xargs)
+DATA_DIR_ORIGIN=$(docker exec -it $CONTAINER_NAME bash -c "$HADOOP_HOME/bin/hdfs getconf -confKey dfs.datanode.data.dir 2>&1" | awk '/file:\/\// {print $1}' | xargs)
+echo "NameNode directory: $NAME_DIR_ORIGIN"
+echo "DataNode directory: $DATA_DIR_ORIGIN"
+
+# 파일 업데이트
+update_file "$HADOOP_HOME/etc/hadoop/tmp/core-site.xml" "$HADOOP_HOME/etc/hadoop/core-site.xml"
+update_file "$HADOOP_HOME/etc/hadoop/tmp/hdfs-site.xml" "$HADOOP_HOME/etc/hadoop/hdfs-site.xml"
+update_file "$HADOOP_HOME/etc/hadoop/tmp/mapred-site.xml" "$HADOOP_HOME/etc/hadoop/mapred-site.xml"
+update_file "$HADOOP_HOME/etc/hadoop/tmp/yarn-site.xml" "$HADOOP_HOME/etc/hadoop/yarn-site.xml"
+
+echo "Configuration files have been copied and updated successfully."
+
+# Get the NameNode directory
+NAME_DIR_CHANGED=$(docker exec -it $CONTAINER_NAME bash -c "$HADOOP_HOME/bin/hdfs getconf -confKey dfs.namenode.name.dir 2>&1" | awk '/file:\/\// {print $1}' | xargs)
+DATA_DIR_CHANGED=$(docker exec -it $CONTAINER_NAME bash -c "$HADOOP_HOME/bin/hdfs getconf -confKey dfs.datanode.data.dir 2>&1" | awk '/file:\/\// {print $1}' | xargs)
+
+# 서비스 재시작
+case $SERVICE_TYPE in
+  namenode)
+    restart_namenode_service $SERVICE_TYPE $NAME_DIR_ORIGIN $NAME_DIR_CHANGED
+    ;;
+  datanode)
+    restart_datanode_service $SERVICE_TYPE
+    ;;
+  resourcemanager|nodemanager)
+    restart_yarn_service $SERVICE_TYPE
+    ;;
+esac
+
+echo "Configuration changes applied and services restarted."

missions/W3/M2/change-config/core-site.xml

@@ -0,0 +1,14 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://namenode:9000</value>
+    </property>
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/hadoop/tmp</value>
+    </property>
+    <property>
+        <name>io.file.buffer.size</name>
+        <value>131072</value>
+    </property>
+</configuration>

missions/W3/M2/change-config/hdfs-site.xml

@@ -0,0 +1,15 @@
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>2</value>
+    </property>
+    <property>
+        <name>dfs.namenode.name.dir</name>
+        <value>file:///hadoop/dfs/name</value>
+    </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>file:///hadoop/dfs/data</value>
+    </property>
+</configuration>
+

missions/W3/M2/change-config/make_dir.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# 인자를 1개만 받는다. 이때 첫 번째 인자는 컨테이너 이름이다.
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 <CONTAINER_NAME>"
+  exit 1
+fi
+
+CONTAINER_NAME=$1
+
+# 함수 정의: 디렉토리 생성
+create_directory() {
+  local dir=$1
+  echo "Creating directory $dir..."
+  docker exec -it $CONTAINER_NAME bin/bash -c "
+    if [ ! -d $dir ]; then
+      sudo mkdir -p $dir && \
+      sudo chown -R hadoop:hadoop $dir
+    fi"
+  if [ $? -ne 0 ]; then
+    echo "Failed to create directory $dir inside the container."
+    exit 1
+  fi
+}
+
+# add hadoop directories for change configure
+create_directory /hadoop/dfs/name
+create_directory /hadoop/dfs/data
+create_directory /hadoop/tmp