#!/bin/bash  software_dir=/root/hadoop/ install_dir=/opt/ hostname=(master node1 node2)  jdk_name=$(tar -tf "$software_dir"jdk*|head -n 1|sed 's|\/||') hadoop_name=$(tar -tf "$software_dir"hadoop*|head -n 1|sed 's|\/||') zookeeper_name=$(tar -tf "$software_dir"zookeeper*|head -n 1|sed 's|\/||')  ### 搭建时间同步集群 echo =================== Start Chrony Install ===================== for host in ${hostname[@]} do ssh -T $host <<-EOF [ ! \$(rpm -qa chrony) ] && yum install -y chrony &>/dev/null [ ! \$(rpm -qa chrony) ] && echo "============ install chrony fail ==============" && exit echo -e  >> /etc/chrony.conf sed -i 's/^server/#server/' /etc/chrony.conf echo -e "server master iburst\nbindaddress ::\nallow" >> /etc/chrony.conf [ ${hostname[0]} = \$(hostname) ] && echo "local stratum 10" >> /etc/chrony.conf && sed -i 's/master/127\.0\.0\.1/' /etc/chrony.conf systemctl enable --now chronyd timedatectl set-timezone Asia/Shanghai chronyc -a makestep exit EOF done [ ! $(rpm -qa chrony) ] && exit  ### xsync script  for host in ${hostname[@]} do ssh -T $host <<-EOF [ ! \$(rpm -qa rsync) ] && yum install -y rsync &>/dev/null [ ! \$(rpm -qa rsync) ] && echo "============ install rsync fail ==============" && exit exit EOF done  [ ! $(rpm -qa rsync) ] && exit  cat > /usr/local/bin/xsync << EOF #!/bin/bash  pcount=\$# if [ \$pcount -lt 1 ] then    echo "Not Enough Arguement !"    exit fi  for host in ${hostname[@]} do   if [ "\$host" = \$(hostname) ]   then       continue   fi   echo ======================== Start \$host File Sync =======================   for file in \$@   do     if [ -e \$file ]         then             pdir=\$(cd -P \$(dirname \$file); pwd)             echo fileDir=\$pdir             fname=\$(basename \$file)             echo fileName=\$fname             ssh \$host "mkdir -p \$pdir"             rsync -av \$pdir/\$fname \$host:\$pdir &>/dev/null         else             echo "\$file does not exists"         fi   done done EOF  chmod +x /usr/local/bin/xsync  ### Install Component(Jdk Hadoop Zookeeper)  echo =================== Start Install Component ===================== tar -xf "$software_dir"jdk* -C $install_dir tar -xf "$software_dir"hadoop* -C $install_dir tar -xf "$software_dir"zookeeper* -C $install_dir  rm -rf $install_dir$hadoop_name/share/doc  xsync ${install_dir}{$jdk_name,$hadoop_name,$zookeeper_name}  for host in ${hostname[@]} do  ssh -T $host <> /etc/profile source /etc/profile exit EOF  done  java -version &>/dev/null [ $? -eq 0 ] && echo "========= java/hadoop/zookeeper/hive installation complete !========"  ### jpsall script cat > /usr/local/bin/jpsall << EOF #!/bin/bash  for host in ${hostname[@]} do   echo -e "\033[32m======================== \$host =======================\033[0m"   if [ $# -gt 0 ]   then     ssh $host "source /etc/profile;$*"    else     ssh $host "source /etc/profile;jps|grep -v Jps"   fi    done EOF  chmod +x /usr/local/bin/jpsall  ### Zookeeper Configuration  echo =================== Start Zookeeper Configuration ===================== zookeeper_path=$install_dir$zookeeper_name  rm -rf $zookeeper_path tar -xf "$software_dir"zookeeper* -C $install_dir  mkdir -p $zookeeper_path/{data,logs} mv $install_dir$zookeeper_name/conf/{zoo_sample.cfg,zoo.cfg}  sed -i "/^dataDir=/c\dataDir=$zookeeper_path/data" $zookeeper_path/conf/zoo.cfg count=1 for host in ${hostname[@]} do    zookeeper_host+="server.$count=$host:2888:3888"    if [ $count -lt $(( ${#hostname[@]} )) ]    then       zookeeper_host+="\n"    fi    ((count++)) done  echo -e "$zookeeper_host" >> $zookeeper_path/conf/zoo.cfg  sed -i  "s|ZOO_LOG_DIR=\".\"|ZOO_LOG_DIR=\"$zookeeper_path/logs\"|" $zookeeper_path/bin/zkEnv.sh  cat > /usr/lib/systemd/system/zookeeper.service < $zookeeper_path/data/myid systemctl daemon-reload systemctl enable --now zookeeper exit EOF ((count++)) done  jpsall "zkServer.sh status"  ### Hadoop Configuration echo =================== Start Hadoop Configuration ===================== hadoop_path=$install_dir$hadoop_name hadoop_conf_path=$hadoop_path/etc/hadoop  stop-all.sh jpsall "rm -rf $hadoop_path" &>/dev/null tar -xf "$software_dir"hadoop* -C $install_dir rm -rf $hadoop_path/share/doc/  cp $hadoop_conf_path/{mapred-site.xml.template,mapred-site.xml} sed -i '/configuration>$/d' $hadoop_conf_path/{core-site,hdfs-site,mapred-site,yarn-site}.xml  for ((i=0; i<${#hostname[@]}; i++))  do   zookeeper_address+=${hostname[$i]}:2181   qjournal_address+=${hostname[$i]}:8485   cluster_node+=${hostname[$i]}   [ $i -lt $(( ${#hostname[@]} -1 )) ] &&  zookeeper_address+="," && qjournal_address+=";" && cluster_node+="\n" done  cat >> $hadoop_conf_path/core-site.xml <            fs.defaultFS     hdfs://hacluster                 hadoop.tmp.dir     $hadoop_path/data                  ha.zookeeper.quorum     $zookeeper_address                  hadoop.proxyuser.root.hosts     *            hadoop.proxyuser.root.groups     *    	  EOF  cat >> $hadoop_conf_path/hdfs-site.xml <            dfs.nameservices     hacluster                 dfs.replication     2                 dfs.ha.namenodes.hacluster     nn1,nn2                 dfs.namenode.rpc-address.hacluster.nn1     ${hostname[0]}:9000                 dfs.namenode.rpc-address.hacluster.nn2     ${hostname[1]}:9000   	              dfs.namenode.http-address.hacluster.nn1     ${hostname[0]}:50070                 dfs.namenode.http-address.hacluster.nn2     ${hostname[1]}:50070                 dfs.namenode.shared.edits.dir     qjournal://$qjournal_address/hacluster                 dfs.journalnode.edits.dir     $hadoop_path/data/dfs/journal                 dfs.ha.fencing.methods     sshfence                 dfs.ha.fencing.ssh.private-key-files     /root/.ssh/id_rsa                 dfs.permissions.enable     false                 dfs.client.failover.proxy.provider.hacluster     org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider                 dfs.ha.automatic-failover.enabled     true      EOF  cat >> $hadoop_conf_path/mapred-site.xml <            mapreduce.framework.name     yarn               mapreduce.jobhistory.address 	${hostname[2]}:10020               mapreduce.jobhistory.webapp.address     ${hostname[2]}:19888        EOF  cat >> $hadoop_conf_path/yarn-site.xml <         yarn.nodemanager.aux-services     mapreduce_shuffle                 yarn.resourcemanager.ha.enabled     true                 yarn.resourcemanager.cluster-id     cluster-yarn                 yarn.resourcemanager.ha.rm-ids     rm1,rm2                 yarn.resourcemanager.hostname.rm1     ${hostname[1]}              yarn.resourcemanager.hostname.rm2     ${hostname[2]}                 yarn.resourcemanager.zk-address     $zookeeper_address                 yarn.resourcemanager.recovery.enable     true                 yarn.resourcemanager.store.class     org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore                  yarn.nodemanager.vmem-check-enabled     false                 yarn.nodemanager.resource.memory-mb     3072                 yarn.log-aggregation-enable     true                 yarn.log-aggregation.retain-seconds     604800               yarn.nodemanager.remote-app-log-dir     /tmp/jobhistory/log        EOF  echo -e $cluster_node > $hadoop_conf_path/slaves  chown -R root:root $hadoop_path sed -i "/^export JAVA_HOME=/c\export JAVA_HOME=$install_dir$jdk_name"  $hadoop_conf_path/hadoop-env.sh   xsync $hadoop_path  for host in ${hostname[@]} do  ssh -T $host </dev/null EOF done  expect <<-EOF   spawn hdfs zkfc -formatZK   expect {      "(Y or N)" {send "Y\r"; exp_continue}     expect eof   } EOF echo ============ ZookeeperCluster Formatting Complete =============  for host in ${hostname[@]} do ssh -T $host </dev/null   if [ \$? -eq 0 ]   then     exit   fi   sleep 4 done EOF done  hdfs namenode -format echo ============ HadoopCluster Formatting Complete =============  hadoop-daemon.sh start namenode  echo ============ Start SecondaryNamenode Data Sync ============= ssh -T ${hostname[1]} <  sed -i -E '/resourcemanager|nodemanager/s/^/#/' $hadoop_path/sbin/$1   cat >> $hadoop_path/sbin/$1 <IDS[\$i]})     if [ \$i -lt \$(( \${#IDS[@]} -1 )) ]     then       NODES+=" "     fi   done   echo "$3 Resourcemanager HA on [\$NODES]"   "\$bin"/yarn-daemons.sh --config \$YARN_CONF_DIR --hostnames "\$NODES" $2 resourcemanager else   "\$bin"/yarn-daemon.sh --config \$YARN_CONF_DIR  $2 resourcemanager	 fi  # $2 nodeManager "\$bin"/yarn-daemons.sh --config \$YARN_CONF_DIR  $2 nodemanager  # $2 historyserver HISTORYSERVER_ENABLE=\$(\$HADOOP_PREFIX/bin/hdfs getconf -confKey yarn.log-aggregation-enable) REMOTE=\$(\$HADOOP_PREFIX/bin/hdfs getconf -confKey mapreduce.jobhistory.webapp.address|cut -d ":" -f1) if [ "\$(echo "\$HISTORYSERVER_ENABLE" | tr A-Z a-z)" = "true" ]; then  echo "$3 Historyserver on [\$REMOTE]"  ssh -T \$REMOTE "\$bin"/mr-jobhistory-daemon.sh $2 historyserver 2>&1 | sed "s/^/\$REMOTE: /" fi EOF } yarn_custom_script start-yarn.sh start Starting yarn_custom_script stop-yarn.sh stop Stoping #### ha集群启动使用该start-all.sh脚本有个bug就启动顺序的问题 该脚本的启动顺序是 namenode->datanode->journal nodes ...... #### 这样执行下来就会导致每次需要执行两次start-all.sh 才能让集群启动成功 #### ha集群正确的启动顺序是 journal nodes -> namenode ->datanode ...... #### 解决方案：把start-all.sh里面关于启动journal node的代码放到namenode 代码前面就能保证ha集群每次启动成功了 sed -i '/^# quor/,/^#------/d;49 r /dev/fd/3' $hadoop_path/sbin/start-dfs.sh \ 3< <(sed -n '/^# quor/,/^#-----/p' $hadoop_path/sbin/start-dfs.sh) sed -i '/Deprecated/s/^/#/' $hadoop_path/sbin/start-all.sh sed -i '/Deprecated/s/^/#/' $hadoop_path/sbin/stop-all.sh start-all.sh jpsall hdfs dfs -chmod -R 777 /tmp echo ============ HadoopCluster Startup Complete =============  #### hive configuration echo ============ Start Hive Configuration ============= hive_path=${install_dir}hive hive_conf_path=$hive_path/conf tez_path=${install_dir}tez  rm -rf $tez_path tar xf ${software_dir}*tez* -C $install_dir mv ${install_dir}*tez* $tez_path chown -R root:root $tez_path  rm -rf $hive_path tar xf ${software_dir}*hive* -C $install_dir mv ${install_dir}*hive* ${install_dir}hive  mysql_user=root mysql_password=1234 if [ -f "$hive_conf_path/hive-log4j2.properties.template" ] then     mv $hive_conf_path/hive-log4j2.properties.template $hive_conf_path/hive-log4j2.properties fi  if [ -f "$hive_conf_path/hive-exec-log4j2.properties.template" ]  then    mv $hive_conf_path/hive-exec-log4j2.properties.template $hive_conf_path/hive-exec-log4j2.properties fi cp ${software_dir}mysql-connector-java-5.1.44-bin.jar $hive_path/lib sed -i "/property.hive.log.dir/c\property.hive.log.dir=$hive_path/logs" $hive_conf_path/hive-log4j2.properties sed -i "/property.hive.log.dir/c\property.hive.log.dir=$hive_path/logs" $hive_conf_path/hive-exec-log4j2.properties  cat > $hive_conf_path/hive-site.xml <               hive.metastore.warehouse.dir     /hive/database               hive.exec.mode.local.auto     true                 hive.resultset.use.unique.column.names     false                   hive.execution.engine     tez               hive.metastore.schema.verification     false            javax.jdo.option.ConnectionURL     jdbc:mysql://${HOSTNAME}:3306/hive?createDatabaseIfNotExist=true&useUnicode=true&characterEncodeing=UTF-8&useSSL=false            javax.jdo.option.ConnectionDriverName     com.mysql.jdbc.Driver            javax.jdo.option.ConnectionUserName     $mysql_user            javax.jdo.option.ConnectionPassword     $mysql_password      EOF rm -rf ${hive_path}/lib/log4j-slf4j-impl-*.jar docker exec -it mysql mysql -u $mysql_user -p$mysql_password -e "drop database if exists hive;" &>/dev/null schematool -dbType mysql -initSchema  hdfs dfs -rm -r /tez hdfs dfs -mkdir /tez tez_name=$(ls $tez_path/share) hdfs dfs -put $tez_path/share/$tez_name /tez rm -rf ${tez_path}/lib/slf4j-log4j12-*.jar  cat > $hive_conf_path/tez-site.xml <           tez.lib.uris     \${fs.defaultFS}/tez/$tez_name            tez.use.cluster.hadoop-libs     true            tez.history.logging.service.class     org.apache.tez.dag.history.logging.ats.ATSHisoryLoggingService        EOF  mv $hive_conf_path/hive-env.sh.template $hive_conf_path/hive-env.sh  cat >> $hive_conf_path/hive-env.sh <