# 大数据软件启动脚本 **Repository Path**: huaiyuechusan/big-data-software-startup-script ## Basic Information - **Project Name**: 大数据软件启动脚本 - **Description**: hadoop，zookeeper，kafka，flume，hive，maxwell，dolphinscheduler，superset，集群一键启动，集群文件分发，集群执行命令，集群同步时间脚本 - **Primary Language**: Unknown - **License**: Not specified - **Default Branch**: master - **Homepage**: None - **GVP Project**: No ## Statistics - **Stars**: 0 - **Forks**: 0 - **Created**: 2023-11-14 - **Last Updated**: 2023-11-20 ## Categories & Tags **Categories**: Uncategorized **Tags**: None ## README # 大数据软件启动脚本 ## hadoop ```shell #!/bin/bash # hadoop集群启动 if [ $# -eq 0 ]; then echo "Usage $0 start|stop" fi case $1 in "start"){ echo " -------- 启动 集群 -------" echo " -------- 启动 hadoop集群 -------" ssh niit01 "source /etc/profile;start-dfs.sh" ssh niit03 "source /etc/profile;start-yarn.sh" ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh start historyserver" };; "stop"){ echo " -------- 停止 集群 -------" echo " -------- 停止 hadoop集群 -------" ssh niit01 "source /etc/profile;mr-jobhistory-daemon.sh stop historyserver" ssh niit03 "source /etc/profile;stop-yarn.sh" ssh niit01 "source /etc/profile;stop-dfs.sh" };; esac ``` ## zookeeper ```shell #!/bin/bash # zookeeper集群启动脚本 echo "Usage: $0 start|stop|status (zookeeper)" case $1 in "start"){ for i in niit01 niit02 niit03 do echo ---------- zookeeper $i 启动 ------------ ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh start" done };; "stop"){ for i in niit01 niit02 niit03 do echo ---------- zookeeper $i 停止 ------------ ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh stop" done };; "status"){ for i in niit01 niit02 niit03 do echo ---------- zookeeper $i 状态 ------------ ssh $i "/opt/onlineedu/zookeeper/bin/zkServer.sh status" done };; esac ``` ## kafka ```shell #!/bin/bash # kafka集群启动 case $1 in "start"){ for i in niit01 niit02 niit03 do echo " --------启动 $i Kafka-------" ssh $i "/opt/onlineedu/kafka/bin/kafka-server-start.sh -daemon /opt/onlineedu/kafka/config/server.properties" done };; "stop"){ for i in niit01 niit02 niit03 do echo " --------停止 $i Kafka-------" ssh $i "/opt/onlineedu/kafka/bin/kafka-server-stop.sh stop" done };; esac ``` ## flume ```shell #! /bin/bash # flume采集通道1启动脚本 case $1 in "start"){ for i in niit01 niit02 do echo " --------启动 $i 采集flume-------" ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/file-flume-kafka.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log1.txt 2>&1 &" done };; "stop"){ for i in niit01 niit02 do echo " --------停止 $i 采集flume-------" ssh $i "ps -ef | grep file-flume-kafka | grep -v grep |awk '{print \$2}' | xargs -n1 kill -9 " done };; esac ``` ```shell #! /bin/bash # flume采集通道2启动脚本 case $1 in "start"){ for i in niit03 do echo " --------启动 $i 消费flume-------" ssh $i "nohup /opt/onlineedu/flume/bin/flume-ng agent --conf-file /opt/onlineedu/flume/job/kafka-flume-hdfs.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/onlineedu/flume/log2.txt 2>&1 &" done };; "stop"){ for i in niit03 do echo " --------停止 $i 消费flume-------" ssh $i "ps -ef | grep kafka-flume-hdfs | grep -v grep | awk '{print \$2}' | xargs -n1 kill -9 " done };; esac ``` ```shell #!/bin/bash # flume采集通道3启动脚本 case $1 in "start") echo " --------启动 niit03 业务数据flume-------" ssh niit03 "nohup /opt/onlineedu/flume/bin/flume-ng agent -n a1 -c /opt/onlineedu/flume/conf -f /opt/onlineedu/flume/job/kafka_to_hdfs_db.conf >/opt/onlineedu/flume/log3.txt >/dev/null 2>&1 &" ;; "stop") echo " --------停止 niit03 业务数据flume-------" ssh niit03 "ps -ef | grep kafka_to_hdfs_db | grep -v grep |awk '{print \$2}' | xargs -n1 kill" ;; esac ``` ## hive ```shell #!/bin/bash # hive启动 HIVE_LOG_DIR=$HIVE_HOME/logs mkdir -p $HIVE_LOG_DIR #检查进程是否运行正常，参数1为进程名，参数2为进程端口 function check_process() { pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}') ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1) echo $pid [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1 } function hive_start() { metapid=$(check_process HiveMetastore 9083) cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &" cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1" [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动" server2pid=$(check_process HiveServer2 10000) cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &" [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动" } function hive_stop() { metapid=$(check_process HiveMetastore 9083) [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动" server2pid=$(check_process HiveServer2 10000) [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动" } case $1 in "start") hive_start ;; "stop") hive_stop ;; "restart") hive_stop sleep 2 hive_start ;; "status") check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常" check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常" ;; *) echo Invalid Args! echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)' ;; esac ``` ## maxwell ```shell #!/bin/bash # hive启动 HIVE_LOG_DIR=$HIVE_HOME/logs mkdir -p $HIVE_LOG_DIR #检查进程是否运行正常，参数1为进程名，参数2为进程端口 function check_process() { pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}') ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1) echo $pid [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1 } function hive_start() { metapid=$(check_process HiveMetastore 9083) cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &" cmd=$cmd" sleep 4; hdfs dfsadmin -safemode wait >/dev/null 2>&1" [ -z "$metapid" ] && eval $cmd || echo "Metastroe服务已启动" server2pid=$(check_process HiveServer2 10000) cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &" [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动" } function hive_stop() { metapid=$(check_process HiveMetastore 9083) [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动" server2pid=$(check_process HiveServer2 10000) [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动" } case $1 in "start") hive_start ;; "stop") hive_stop ;; "restart") hive_stop sleep 2 hive_start ;; "status") check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常" check_process HiveServer2 10000 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常" ;; *) echo Invalid Args! echo 'Usage: '$(basename $0)' start|stop|restart|status (hiveserver2 & metastore)' ;; esac ``` ## dolphinscheduler ```shell #!/bin/bash # dolphinscheduler集群启动 function node_start() { /opt/onlineedu/dolphinscheduler/bin/start-all.sh /home/niit/bin/xRun.sh jps } function node_stop() { /opt/onlineedu/dolphinscheduler/bin/stop-all.sh /home/niit/bin/xRun.sh jps } case $1 in "start") node_start ;; "stop") node_stop ;; "restart") node_stop sleep 10 node_start ;; *) esac ``` ```shell #!/bin/bash # dolphinscheduler单机版启动 function node_start() { /opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh start standalone-server /home/niit/bin/xRun.sh jps } function node_stop() { /opt/onlineedu/dolphinscheduler/bin/dolphinscheduler-daemon.sh stop standalone-server /home/niit/bin/xRun.sh jps } case $1 in "start") node_start ;; "stop") node_stop ;; "restart") node_stop sleep 10 node_start ;; *) esac ``` ## superset ```shell #!/bin/bash # superset可视化启动 superset_status(){ result=`ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | wc -l` if [[ $result -eq 0 ]]; then return 0 else return 1 fi } superset_start(){ source ~/.bashrc superset_status >/dev/null 2>&1 if [[ $? -eq 0 ]]; then conda activate superset ; gunicorn --workers 5 --timeout 120 --bind niit01:8787 --daemon 'superset.app:create_app()' else echo "superset正在运行" fi } superset_stop(){ superset_status >/dev/null 2>&1 if [[ $? -eq 0 ]]; then echo "superset未在运行" else ps -ef | awk '/gunicorn/ && !/awk/{print $2}' | xargs kill -9 fi } case $1 in start ) echo "启动Superset" superset_start ;; stop ) echo "停止Superset" superset_stop ;; restart ) echo "重启Superset" superset_stop superset_start ;; status ) superset_status >/dev/null 2>&1 if [[ $? -eq 0 ]]; then echo "superset未在运行" else echo "superset正在运行" fi esac ``` ## 集群一键启动 ```shell #!/bin/bash # 所有集群所有软件启动脚本 function node_start() { /home/niit/bin/xCluster.sh start sleep 15 /home/niit/bin/zk.sh start sleep 10 /home/niit/bin/kf.sh start sleep 10 /home/niit/bin/hv.sh start sleep 5 /home/niit/bin/mxw.sh start sleep 5 /home/niit/bin/f1.sh start sleep 5 /home/niit/bin/f2.sh start sleep 5 /home/niit/bin/f3.sh start sleep 5 /home/niit/bin/dolphinscheduler-standalone.sh stop sleep 5 /home/niit/bin/superset.sh start sleep 5 /home/niit/bin/xRun.sh jps } function node_stop() { /home/niit/bin/mxw.sh stop sleep 5 /home/niit/bin/f1.sh stop sleep 5 /home/niit/bin/f2.sh stop sleep 5 /home/niit/bin/f3.sh stop sleep 5 /home/niit/bin/kf.sh stop sleep 10 /home/niit/bin/kf.sh stop sleep 10 /home/niit/bin/hv.sh stop sleep 5 /home/niit/bin/xCluster.sh stop sleep 15 /home/niit/bin/zk.sh stop sleep 10 /home/niit/bin/superset.sh stop sleep 5 /home/niit/bin/dolphinscheduler-standalone.sh stop sleep 5 /home/niit/bin/xRun.sh jps } case $1 in "start") node_start ;; "stop") node_stop ;; "restart") node_stop sleep 10 node_start ;; *) esac ``` ## 集群文件分发 ```shell #!/bin/bash # 集群分发文件脚本，普通用户权限 #1. 判断参数个数 if [ $# -lt 1 ] then echo Not Enough Arguement! echo Usage: xSync.sh path1 path2.. exit; fi #2. 遍历集群所有机器 for host in niit01 niit02 niit03 do echo ==================== $host ==================== #3. 遍历所有目录，挨个发送 for file in $@ do #4 判断文件是否存在 if [ -e $file ] then #5. 获取父目录 pdir=$(cd -P $(dirname $file); pwd) #6. 获取当前文件的名称 fname=$(basename $file) ssh $host "mkdir -p $pdir" rsync -av $pdir/$fname $host:$pdir else echo $file does not exists! fi done done ``` ```shell #!/bin/bash # 集群分发文件脚本，root权限 #1. 判断参数个数 if [ $# -lt 1 ] then echo Not Enough Arguement! echo Usage: xSync.sh path1 path2 ... exit; fi #2. 遍历集群所有机器 for host in niit01 niit02 niit03 do echo ==================== $host ==================== #3. 遍历所有目录，挨个发送 for file in $@ do #4 判断文件是否存在 if [ -e $file ] then #5. 获取父目录 pdir=$(cd -P $(dirname $file); pwd) #6. 获取当前文件的名称 fname=$(basename $file) sudo ssh $host "mkdir -p $pdir" sudo rsync -av $pdir/$fname $host:$pdir else echo $file does not exists! fi done done ``` ## 集群执行命令 ```shell #/bin/bash # 集群执行命令脚本 if (( $# == 0 ));then echo "Usage xRun.sh \"\"" exit 0 fi for node in niit01 niit02 niit03 do echo "======== $node ========" echo "ssh $node $1" ssh $node "source /etc/profile;$1" done ``` ## 集群同步时间 ```shell #!/bin/bash # 集群时间同步脚本，需要安装chrony到集群中 # 检查参数是否为空 if [ -z "$1" ]; then echo "Usage: `basename $0` yyyy-MM-dd HH:mm:ss" exit 1 fi # 使用date命令将时间字符串转换为日期和时间 # 如果转换失败，则说明时间字符串不合法 if ! date -d "$*" >/dev/null 2>&1; then echo "Wrong argument for $*" echo "Usage: `basename $0` yyyy-MM-dd HH:mm:ss" exit 1 fi echo ">>>>>>>>>>>> SYNC TIME START >>>>>>>>>>>>" sum=-1 while [ $sum -ne 0 ]; do echo set time for niit01 to $1 '>>>' ssh niit01 "(sudo timedatectl set-ntp false && sudo date -s \"$*\" && sudo timedatectl set-ntp true)" ok1=$? echo sync time from niie02 to niit01 '>>>' ssh niit02 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)" ok2=$? echo sync time from niit03 to niit01 '>>>' ssh niit03 "(sudo timedatectl set-ntp false && sudo timedatectl set-ntp true)" ok3=$? sum=`expr $ok1 + $ok2 + $ok3` if [ $sum -eq 0 ]; then echo "<<<<<<<<<<<<< SYNC TIME END <<<<<<<<<<<<<" sleep 5 xRun.sh date else echo "sync time failed, will try 10 senconds later" sleep 10 fi done ```