集群指令集合

node1 node2 node3
nn1 nn2 dn3
dn1 dn2 nm3
rm1 rm2 zk3
nm1 nm2 MysqL
zk1 zk2
hivestat hivserv hiveMeta

zkServer.sh start

主节点启动： start-dfs.sh
#主节点启动： yarn-daemon.sh start resourcemanager
主节点启动： start-yarn.sh
stop-yarn.sh stop-dfs.sh

hive --service metastore > /home/hadoop/hive.Meta &
hive --service hiveserver2 > /home/hadoop/hive.log &

#hadoop fs -mkdir -p /user/spark/libs/
#hadoop fs -put /data/software/spark-2.1.1/jars/* /user/spark/libs/
hadoop fs -mkdir -p /tmp/spark/logs/

start-master.sh
start-slaves.sh

zkCli.sh
rm -rf /data/software/spark-2.1.1/conf/
scp -r /data/software/spark-2.1.1/conf/ hadoop@app-002:/data/software/spark-2.1.1/

Yarn运行日志：
/tmp/logs/hadoop/logs

 

提交任务做测试：

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar pi 1 3

spark-shell --master yarn --deploy-mode client

spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode cluster \
--driver-memory 1000m \
--executor-memory 1000m \
--executor-cores 1 \
/data/software/spark-2.1.1/examples/jars/spark-examples_2.11-2.1.1.jar \
3

 

 

debug:

nohup java -jar sent-mail.jar > log.txt &

查看端口是否占用：netstat -ntulp |grep 8020
查看一个服务有多少端口：ps -ef |grep MysqLd

rm -rf /data/software/hadoop-2.7.3/logs/*

单独启动各个组件， 查看bug产生原因。
hadoop-daemon.sh start namenode
hadoop-daemon.sh start datanode

jps -ml
kill -9

 

shell-脚本：