hadoop的yarn部署

MapReduce配置文件

mapred-env.sh,新增环境变量

[root@hadoop1 hadoop]# vi /export/server/hadoop/etc/hadoop/mapred-env.sh
export JAVA_HOME=/export/server/jdk
export HAPOOD_JOB_HISTORYSERVER_HEAPSIZE=1000
export HAPOOD_MAPRED_ROOT_LOGGER=INFO,RFA

mapred-site.xml,新增环境变量

[root@hadoop1 hadoop]# vi /export/server/hadoop/etc/hadoop/mapred-site.xml
        
                mapreduce.framework.name
                yarn
                运行框架设置
        
        
                mapreduce.jobhistory.address
                hadoop1:10020
                历史服务器通讯端口
        
        
                mapreduce.jobhistory.webapp.address
                hadoop1:19888
                历史服务器web地址
        
        
                mapreduce.jobhistory.intermediate-done-dir
                /data/mspReduceHistory/temp
                历史信息在hdfs的记录路径
        
        
                yarn.app.mapreduce.am.env
                /data/mspReduceHistory/done
                历史信息在hdfs的记录路径
        
        
                yarn.app.mapreduce.am.env
                HADOOP_MAPRED_HOME=$HADOOP_HOME
                MapReduce Home 设置为 HADOOP_HOME
        
        
                mapreduce.map.env
                HADOOP_MAPRED_HOME=$HADOOP_HOME
                MapReduce Home 设置为 HADOOP_HOME
        
        
                mapreduce.reduce.env
                HADOOP_MAPRED_HOME=$HADOOP_HOME
                MapReduce Home 设置为 HADOOP_HOME
        

yarn配置文件

yarn-env.sh

[root@hadoop1 hadoop]# mkdir -p /data/hadoopLog
[root@hadoop1 hadoop]# vi /export/server/hadoop/etc/hadoop/yarn-env.sh
export JAVA_HOME=/export/server/jdk
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=/data/hadoopLog

yarn-site.xml

[root@hadoop1 data]# mkdir -p /data/nodeManagerLocal
[root@hadoop1 data]# mkdir -p /data/nodeManagerLog
[root@hadoop1 data]# vi /export/server/hadoop/etc/hadoop/yarn-site.xml
	
		yarn.resourcemanager.hostname
		hadoop1
		ResourceManager设置节点
	
	
		yarn.nodemanager.local-dirs
		/data/nodeManagerLocal
		ResourceManager中间数据库本地存储路径
	
	
		yarn.nodemanager.log-dirs
		/data/nodeManagerLog
		ResourceManager数据日志本地存储路径
	
	
		yarn.nodemanager.aux-services
		mapreduce_shuffle
		为MapReduce程序开启Shuffle服务
	
	
		yarn.log.server.url
		http://hadoop1:19888/jobhistory/log
		历史服务器url
	
	
		yarn.web-proxy.address
		hadoop1:8089
		代理服务器地址
	
	
		yarn.log-aggregation-enable
		true
		开始日志聚合
	
	
		yarn.nodemanager.remote-app-log-dir
		/tmp/logs
		程序日志hdfs的存储路径
	
	
		yarn.resourcemanager.scheduler.class
		org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
		选择公平调度器
	
	

同步到hadoop2,hadoop3主机

[root@hadoop1 hadoop]# cd /export/server/hadoop/etc/hadoop
[hadoop@hadoop1 hadoop]$ scp mapred-env.sh mapred-site.xml yarn-env.sh yarn-site.xml hadoop2:`pwd`/
[hadoop@hadoop1 hadoop]$ scp mapred-env.sh mapred-site.xml yarn-env.sh yarn-site.xml hadoop3:`pwd`/

集群启动命令

chown -R hadoop:hadoop /data
#一键启动yarn集群
start-yarn.sh
#一键停止yarn集群
stop-yarn.sh
#在当前机器，单独启动和停止进程
yarn --daemon start|stop resourcemanager|nodemanager|proxyserver
#历史服务器启动和停止
mapred --deamon start|stop historyserver