Linux下安装Hadoop集群（2.部署Hadoop）

1.下载解压hadoop2.7.1

cd /usr/local
wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.1/hadoop-2.7.1.tar.gz
tar -zxvf hadoop-2.7.1.tar.gz

嫌速度慢的话可以下载这个（不要脸.jpg）
hadoop2.7.1下载

2.配置环境变量

vi /etc/profile
#添加如下配置
export HADOOP_HOME=/usr/local/hadoop-2.7.1
export PATH=$PATH:$HADOOP_HOME/bin
#使配置生效
source /etc/profile
#测试（输入hadoop有提示则表示成功）
hadoop
#将profile发送到其他节点
scp /etc/profile [e-mail]:/etc
scp /etc/profile [e-mail]:/etc
#记得在其他节点下source
source /etc/profile

3.修改core-site.xml(配置Service的URL地址、Hadoop集群的临时目录等信息)

#在usr/local下创建hdfs目录
cd /usr/local 
mkdir hdfs
cd hdfs
mkdir name
mkdir data
mkdir tmp
#将hdfs文件夹传到其他两个节点，注意权限问题
chmod 777 usr/local #其他节点也要
chmod 777 usr/local/hdfs
scp -r /usr/local/hdfs [e-mail]:/usr/local
scp -r /usr/local/hdfs [e-mail]:/usr/local
#进入配置文件路径
cd /usr/local/hadoop-2.7.1/etc/hadoop
vi core-site.xml

core-site.xml的配置如下

<configuration>
 <property>
   <name>hadoop.tmp.dir</name>
   <value>file:/usr/local/hdfs/tmp</value> 
  </property>
 <property>
   <name>io.file.buffer.size</name>
   <value>131072</value>
 </property>
 <property>
   <name>fs.default.name</name>
   <value>hdfs://dn1:9000</value>
 </property>
 <property>
   <name>hadoop.proxyuser.root.hosts</name>
   <value>*</value>
 </property>
 <property>
   <name>hadoop.proxyuser.root.groups</name>
   <value>*</value>
 </property>
</configuration>

4.修改hdfs-site.xml

vi hdfs-site.xml

hdfs-site.xml的配置如下

<configuration>
 <property>
   <name>dfs.replication</name>
   <value>2</value>
 </property>
 <property>
   <name>dfs.namenode.name.dir</name>
   <value>file:/usr/local/hdfs/name</value>
   <final>true</final>
 </property>
 <property>
   <name>dfs.datanode.data.dir</name>
   <value>file:/usr/local/hdfs/data</value>
   <final>true</final>
 </property>
 <property>
   <name>dfs.namenode.secondary.http-address</name>
   <value>dn1:9001</value>
 </property>
 <property>
   <name>dfs.webhdfs.enabled</name>
   <value>true</value>
 </property>
 <property>
   <name>dfs.permissions</name>
   <value>false</value>
 </property>
</configuration>

5.添加修改mapred-site.xml

#将mapred-site.xml.template复制为mapred-site.xml
cp mapred-site.xml.template  mapred-site.xml

mapred-site.xml的配置如下

<!--计算任务托管的资源框架名称-->
 <property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
 </property>

6. 修改vim yarn-site.xml

vi yarn-site.xml

yarn-site.xml的配置如下

<configuration>

<!-- Site specific YARN configuration properties -->
 <property>
   <name>yarn.resourcemanager.address</name>
   <value>dn1:8132</value>
 </property>
 <property>
   <name>yarn.resourcemanager.scheduler.address</name>
   <value>dn1:8130</value>
 </property>
 <property>
   <name>yarn.resourcemanager.webapp.address</name>
   <value>dn1:8188</value>
 </property>
 <property>
   <name>yarn.resourcemanager.resource-tracker.address</name>
   <value>dn1:8131</value>
 </property>
 <property>
   <name>yarn.resourcemanager.admin.address</name>
   <value>dn1:8033</value>
 </property>
 <property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
 </property>

 <property>
   <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
   <value>org.apache.hadoop.mapred.ShuffleHandler</value>
 </property>
  <property>
   <name>yarn.nodemanager.resource.memory-mb</name>
   <value>2048</value>
 </property>
 <property>
   <name>yarn.nodemanager.resource.cpu-vcores</name>
   <value>1</value>
 </property>
</configuration>

7.修改slaves和hadoop-env.sh文件

#修改slaves文件
vi /usr/local/hadoop-2.7.1/etc/hadoop/slaves
#将localhost改成下方
dn2
dn3
#修改hadoop-env.sh文件(注释掉之前的，添加完整路径)
vi /usr/local/hadoop-2.7.1/etc/hadoop/hadoop-env.sh 
#export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/local/jdk1.8.0_231

8.将hadoop2.7.1的文件夹复制到dn2和dn3

scp -r /usr/local/hadoop-2.7.1 [e-mail]:/usr/local
scp -r /usr/local/hadoop-2.7.1 [e-mail]:/usr/local

运行Hadoop

#格式化
hadoop namenode -format
cd /usr/local/hadoop-2.7.1/sbin/
#启动hadoop
sh ./start-all.sh
#查看集群状态
/usr/local/hadoop-2.7.1/bin/hdfs dfsadmin -report

#测试yarm，登录 地址是（yarn.resourcemanager.webapp.address）的value
http://192.168.65.77:8188/cluster/cluster

#测试HDFS
http://192.168.65.77:50070/dfshealth.html