docker 部署 hadoop集群
docker 安装
curl -sSL https://get.daocloud.io/docker | sh
##换源
curl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s http://67e93489.m.daocloud.io
systemctl restart docker拉取镜像
docker pull docker pull centos
# 查看镜像是否拉取成功
docker images创建容器
#设置固定IP
docker network create --subnet=172.16.0.0/16 huan-net
#创建容器
#cluster-master
docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-master -h cluster-master --net netgroup --ip 172.16.0.2 daocloud.io/library/centos /usr/sbin/init
#cluster-slaves
docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave1 -h cluster-slave1 --net netgroup --ip 172.16.0.3 daocloud.io/library/centos /usr/sbin/init
docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave2 -h cluster-slave2 --net netgroup --ip 172.16.0.4 daocloud.io/library/centos /usr/sbin/init
docker run -d --privileged -ti -v /sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave3 -h cluster-slave3 --net netgroup --ip 172.16.0.5 daocloud.io/library/centos /usr/sbin/init更换YUM源
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo部署Openssh
master
yum -y install openssh openssh-server openssh-clients
systemctl start sshd
vi /etc/ssh/ssh_config
#设置StrictHostKeyChecking为no
systemctl restart sshdslave1
yum -y install openssh openssh-server openssh-clients
systemctl start sshdslave2
yum -y install openssh openssh-server openssh-clients
systemctl start sshdslave3
yum -y install openssh openssh-server openssh-clients
systemctl start sshd公钥分发
ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub <romte_ip>zookeeper配置
master
将文件从主机复制到docker容器
sudo docker cp apache-zookeeper-3.5.5.tar.gz cluster-slave1:/root/tar
创建目录放置解压文件
mkdir /opt/zookeeper
解压
tar -xivf apache-zookeeper-3.5.5.tar.gz -C /opt/zookeeper/
创建zoo.cfg配置文件
cp /opt/zookeeper/apache-zookeeper-3.5.5/conf/zoo.cfg /opt/zookeeper/apache-zookeeper-3.5.5/conf/zoo.cfg
编辑zoo配置文件
vim /opt/zookeeper/apache-zookeeper-3.5.5/conf/zoo.cfg
添加结点参数
server.1=172.16.0.2:2888:3888
server.2=172.16.0.3:2899:3899
server.3=172.16.0.4:2877:3877创建myid文件
echo 1 > /tmp/zookeeper/myid
编辑环境变量配置文件
vim ~/.bash_profile
添加环境变量
ZK_HOME=/opt/zookeeper/apache-zookeeper-3.5.5
PATH=$PATH:$ZK_HOME/bin使配置生效
source ~/.bash_profile
将配置文件发送到slave1,slave2
scp -r ~/.bash_profile cluster-slave1:~/
scp -r ~/.bash_profile cluster-slave2:~/
scp -r /tmp/zookeeper/myid cluster-slave1:/tmp/zookeeper/
scp -r /tmp/zookeeper/myid cluster-slave2:/tmp/zookeeper/
scp -r zoo.cfg cluster-slave1:/opt/zookeeper/apache-zookeeper-3.5.5/conf/
scp -r zoo.cfg cluster-slave2:/opt/zookeeper/apache-zookeeper-3.5.5/conf/
slave1
使配置生效
source ~/.bash_profile
修改myid文件为2
slave2
修改myid文件为3
启动
zhServer.sh start
Hadoop配置
core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://jinbill</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>192.168.11.46:12181</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/hadoop</value>
</property>
</configuration>hdfs-site.xml
<configuration>
<property>
<name>dfs.nameservices</name>
<value>jinbill</value>
</property>
<property>
<name>dfs.ha.namenodes.jinbill</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.jinbill.nn1</name>
<value>cluster-master:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.jinbill.nn2</name>
<value>cluster-slave1:8020</value>
</property>
<property>
<name>dfs.namenode.http-address.shsxt.nn1</name>
<value>cluster-master:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.shsxt.nn2</name>
<value>cluster-slave1:50070</value>
</property>
<property>
<!-- 指定 namenode 元数据存储在 journalnode 中的路径 -->
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://cluster-slave1:8485;cluster-slave2:8485;cluster-slave3:8485/jinbill</value>
</property>
<property>
<!-- 指定 HDFS 客户端连接 active namenode 的 java 类 -->
<name>dfs.client.failover.proxy.provider.jinbill</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
</value>
</property>
<property>
<!-- 配置隔离机制为 ssh 防止裂 脑裂 -->
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<!-- 指定秘钥的位置 -->
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<!-- 指定 journalnode 日志文件存储的路径 -->
<name>dfs.journalnode.edits.dir</name>
<value>/opt/hadoop/data</value>
</property>
<property>
<!-- 开启自动故障转移 -->
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
</configuration>yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>mr_jinbill</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>cluster-slave2</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>cluster-slave3</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>192.168.11.46:12181</value>
</property>
</configuration>mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>slaves
cluster-slave1
cluster-slave2
cluster-slave3启动三个JornalNode
hadoop-daemon.sh start journalnode
在其中一个NameNode上格式化
hdfs namenode -format
把格式化后的元数据拷贝到另一个NameNode上
元数据初始化后会存放在
hadoop.tmp.dir这个参数配置的地址下的叫dfs的目录里面，将这个目录复制到另一个NN的同目录下
启动刚刚格式化的NameNode
hadoop-daemon.sh start namenode
在没有格式化的NameNode上执行
hdfs namenode -bootstrapStandby
启动第二个NameNode
hadoop-daemon.sh start namenode
在其中一个NameNode上初始化zkfc
hdfs zkfc –formatZK
停止所有结点
stop-dfs.sh
全面启动
start-dfs.sh
start-yarn.sh
访问地址 50070端口和8088端口
因为我是跨域访问，所以用cmd添加一个路由
route add 172.16.0.0 mask 255.255.0.0 192.168.11.38 -p
- 网段地址
- 子网掩码
- 虚拟机地址，docker容器部署在该虚拟机上
效果展示
腾讯云开发者
