手把手教你在腾讯云上搭建hadoop3.x伪集群的方法

$ cd /home/centos
$ mkdir software
$ mkdir module

$ cd software
# 然后把文件拖进去即可

$ tar -zxvf jdk-8u212-linux-x64.tar.gz -C ../module
$ tar -zxvf hadoop-3.1.3.tar.gz -C ../module

$ cd /etc/profile.d/
$ vim my_env.sh

#JAVA_HOME,PATH 
# export 提升为全局变量，如果你的路径和我不同，记得这里要使用自己的路径哦
export JAVA_HOME=/home/centos/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME
export HADOOP_HOME=/home/centos/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

$ source /etc/profile

$ hadoop version

$ java

$ ssh-keygen -t rsa

$ ssh-copy-id 本机hostname

vi /etc/hosts
#这里我保留的配置为，其中master配置的是腾讯云的内网， 如果配置外网会导致eclipse客户端连不上hadoop
::1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
172.16.0.3 master
127.0.0.1 localhost

vi /etc/sysconfig/network 
#修改HOSTNAME为master
HOSTNAME=master

$ hostnamectl --static set-hostname master

$ systemctl disable firewalld    #永久

$ cd /home/centos/module/hadoop-3.1.3/etc/hadoop

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://腾讯云内网ip地址:9820</value>
  </property>
 
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/home/centos/module/hadoop-3.1.3/data/tmp</value>
  </property>
	<!-- 通过web界面操作hdfs的权限 -->
  <property>
    <name>hadoop.http.staticuser.user</name>
    <value>root</value>
  </property>
    <!-- 后面hive的兼容性配置 -->
  <property>
    <name>hadoop.proxyuser.root.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.root.groups</name>
    <value>*</value>
    </property>
</configuration>

<configuration>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>腾讯云内网ip地址:9868</value>
  </property>
</configuration>

export JAVA_HOME=/home/centos/module/jdk1.8.0_212

<configuration>

  <!-- Reducer获取数据的方式-->
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <!-- 指定YARN的ResourceManager的地址-->
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>master</value>
  </property>
<!-- 环境变量通过从NodeManagers的容器继承的环境属性，对于mapreduce应用程序，除了默认值 hadoop op_mapred_home应该被添加外。属性值 还有如下-->
  <property>
    <name>yarn.nodemanager.env-whitelist</name>
 <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
    <!-- 解决Yarn在执行程序遇到超出虚拟内存限制，Container被kill -->
  <property>
    <name>yarn.nodemanager.pmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
  </property>
    <!-- 后面hive的兼容性配置 -->
  <property>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>512</value>
  </property>
  <property>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>4096</value>
  </property>
  <property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>4096</value>
    </property>
  
  <!-- 开启日志聚集 -->
  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
  </property>
  <!-- 访问路径-->
  <property> 
    <name>yarn.log.server.url</name> 
    <value>http://172.17.0.13:19888/jobhistory/logs</value>
  </property>
  <!-- 保存的时间7天 -->
  <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
  </property>
</configuration>

<!-- 历史服务器端地址 -->
<property>
  <name>mapreduce.jobhistory.address</name>
  <value>腾讯云内网ip:10020</value>
</property>

<!-- 历史服务器web端地址 -->
<property>
  <name>mapreduce.jobhistory.webapp.address</name>
  <value>腾讯云内网ip:19888</value>
</property>

$ hdfs namenode -format

$ start-dfs.sh

[root@VM_0_13_centos hadoop]# jps
20032 Jps
30900 DataNode
31355 SecondaryNameNode
30559 NameNode

$ vim mycluster

#!/bin/bash
case $1 in
"start")
#dfs yarn history
start-dfs.sh
start-yarn.sh
mapred --daemon start historyserver
;;
"stop")
# dfs yarn history
stop-dfs.sh
stop-yarn.sh
mapred --daemon stop historyserver
;;
*)
echo "args is error! please input start or stop"
;;
esac

$ chmod u+x mycluster

$ mycluster start

$ jps
23680 NodeManager
24129 JobHistoryServer
22417 DataNode
24420 Jps
22023 NameNode
23384 ResourceManager
22891 SecondaryNameNode

$ mycluster stop

$ vim /home/centos/module/hadoop-3.1.3/share/hadoop/hdfs/webapps/static/dfs-dust.js

return new Date(Number(v)).toLocaleString();

$ mycluster start

$ mkdir temdatas

$ vim text.txt

$ hdfs dfs -put text.txt /

$ hdfs dfs -get /text.txt ./text1.txt

#或者自己在vim中写好上传也可
$ hdfs dfs -put wordcount.txt /input

$ hadoop jar /home/centos/module/hadoop-3.1.3/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount /input /output

#拉取hdfs文件
[root@master mydata]# hdfs dfs -get /output ./
#查看结果
[root@master output]# cat part-r-00000 
a    2
b    3
c    2
d    1
e    1
f    1