wget https://***/packages/29/alluxio-2.5.0-bin.tar.gz (请联系作者询问cos地址)
# md5
[root@172 ~]# md5sum alluxio-2.5.0-bin.tar.gz
127b0ea8616ed14bb68a4be7a5ffd0a2 alluxio-2.5.0-bin.tar.gz
# 备份181
mv /usr/local/service/alluxio{,.181}
# 解压250
tar -xvf /home/hadoop/alluxio-2.5.0-bin.tar.gz -C /usr/local/service/
mv /usr/local/service/alluxio-2.5.0/ /usr/local/service/alluxio
# 替换配置
cd /usr/local/service/; mv alluxio/conf{,.bak}; cp -r alluxio.181/conf alluxio/; cd
mkdir /etc/alluxio
ln -s /usr/local/service/alluxio/conf/alluxio-site.properties /etc/alluxio/alluxio-site.properties
chown -R hadoop:hadoop /etc/alluxio
# 修改/usr/local/service/alluxio/bin/alluxio-start.sh
######################################
510 master)
511 start_master "${FORMAT}"
512 start_job_master # 此行为新增
513 ;;
537 worker)
538 start_worker "${MOPT}"
539 start_job_worker # 此行为新增
540 ;;
######################################
# 修改/usr/local/service/alluxio/bin/alluxio-stop.sh
######################################
59 stop_master() {
60 if [[ ${ALLUXIO_MASTER_SECONDARY} == "true" ]]; then
61 ${LAUNCHER} "${BIN}/alluxio" "killAll" "alluxio.master.AlluxioSecondaryMaster"
62 else
63 ${LAUNCHER} "${BIN}/alluxio" "killAll" "alluxio.master.AlluxioMaster"
64 stop_job_master # 此行为新增
65 fi
66 }
81 stop_worker() {
82 ${LAUNCHER} "${BIN}/alluxio" "killAll" "alluxio.worker.AlluxioWorker"
83 stop_job_worker # 此行为新增
84 }
######################################
# 升级其他组件依赖的alluxio client
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | awk -F"alluxio-" '{print $1}' > /root/path.log
while read line; do cp /usr/local/service/alluxio/client/alluxio-2.5.0-client.jar $line; chown -R hadoop:hadoop $line; done < /root/path.log
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | xargs rm -rf
find /usr/local/service/ -name alluxio-*-client.jar
wget https://woodpecker-cos-apps-bj-1259353343.cos.ap-beijing.myqcloud.com/packages/29/tez-0.9.2.tar.gz
# md5
[root@172 ~]# md5sum tez-0.9.2.tar.gz
47fc7b120b31bc473d62d18ecc73d746 tez-0.9.2.tar.gz
# 替换
mv /usr/local/service/tez/tez-0.9.2.tar.gz{,.bak}
cp /root/tez-0.9.2.tar.gz /usr/local/service/tez/
chown -R hadoop:hadoop /usr/local/service/tez/tez-0.9.2.tar.gz
find /usr/local/service/ -name tez-0.9.2.tar.gz | xargs ls -l
mv /usr/local/service/alluxio{,.181}
mv /usr/local/service/tez/tez-0.9.2.tar.gz{,.bak}
scp -r /usr/local/service/alluxio $OTHER_NODE:/usr/local/service/
scp /usr/local/service/tez/tez-0.9.2.tar.gz $OTHER_NODE:/usr/local/service/tez/
mkdir /etc/alluxio
ln -s /usr/local/service/alluxio/conf/alluxio-site.properties /etc/alluxio/alluxio-site.properties
chown -R hadoop:hadoop /usr/local/service/tez/tez-0.9.2.tar.gz /etc/alluxio /usr/local/service/alluxio
# 升级其他组件依赖的 alluxio client
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | awk -F"alluxio-" '{print $1}' > /root/path.log
while read line; do cp /usr/local/service/alluxio/client/alluxio-2.5.0-client.jar $line; chown -R hadoop:hadoop $line; done < /root/path.log
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | xargs rm -rf
find /usr/local/service/ -name alluxio-*-client.jar
# 首先需将除本台master之外的其他master、core节点ip写入 /root/nodelist 中(换行分割)
while read line; do ssh-copy-id $line; done < /root/nodelist
for line in `cat /root/nodelist`
do
ssh $line "mv /usr/local/service/alluxio{,.181}; mv /usr/local/service/tez/tez-0.9.2.tar.gz{,.bak}; mkdir /etc/alluxio"
echo $line
scp -r /usr/local/service/alluxio $line:/usr/local/service/
scp /usr/local/service/tez/tez-0.9.2.tar.gz $line:/usr/local/service/tez/
ssh $line "ln -s /usr/local/service/alluxio/conf/alluxio-site.properties /etc/alluxio/alluxio-site.properties; chown -R hadoop:hadoop /usr/local/service/tez/tez-0.9.2.tar.gz /etc/alluxio /usr/local/service/alluxio; find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 > /root/path.log"
done
# 替换alluxio-*-client.jar
while read line; do targetPath=`echo ${line%/*}`; cp /usr/local/service/alluxio/client/alluxio-2.5.0-client.jar $targetPath; chown -R hadoop:hadoop $targetPath; done < /root/path.log
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | xargs rm -rf
find /usr/local/service/ -name alluxio-*-client.jar
Tips: 替换jar包(最后三行)需登录其他节点一一操作,批量有点问题。
#EMR PROFILE ENV
source /etc/profile
# Common properties
alluxio.master.hostname=172.21.16.35
alluxio.underfs.address=hdfs://HDFS84486
# Security properties
# alluxio.security.authorization.permission.enabled=true
# alluxio.security.authentication.type=SIMPLE
# Worker properties
alluxio.worker.memory.size=8GB
alluxio.worker.tieredstore.levels=1
alluxio.worker.tieredstore.level0.alias=MEM
alluxio.worker.tieredstore.level0.dirs.path=/mnt/alluxio_used_and_do_not_delete_me
alluxio.worker.tieredstore.level0.watermark.high.ratio=0.95
alluxio.worker.tieredstore.level0.watermark.low.ratio=0.7
alluxio.worker.evictor.class=alluxio.worker.block.evictor.LRUEvictor
alluxio.worker.evictor.lrfu.attenuation.factor=1
alluxio.worker.evictor.lrfu.step.factor=0.25
# User properties
alluxio.user.file.readtype.default=CACHE_PROMOTE
alluxio.user.block.size.bytes.default=64M
alluxio.user.file.writetype.default=MUST_CACHE # ALLUXIO_USER_FILE_WRITETYPE_DEFAULT
alluxio.user.file.copyfromlocal.write.location.policy.class=alluxio.client.file.policy.RoundRobinPolicy
alluxio.user.file.write.location.policy.class=alluxio.client.file.policy.LocalFirstPolicy
alluxio.user.ufs.block.read.location.policy=alluxio.client.file.policy.LocalFirstPolicy
alluxio.user.ufs.block.read.location.policy.deterministic.hash.shards=1
alluxio.user.file.passive.cache.enabled=true # ALLUXIO_USER_FILE_PASSIVE_CACHE_ENABLED
alluxio.zookeeper.enabled=true
alluxio.zookeeper.address=172.21.16.49:2181,172.21.16.42:2181,172.21.16.88:2181
alluxio.master.journal.folder=hdfs://HDFS84486/data/alluxio/journal
# 修改的配置项
# User properties
alluxio.user.file.writetype.default=ASYNC_THROUGH # ALLUXIO_USER_FILE_WRITETYPE_DEFAULT
alluxio.user.file.passive.cache.enabled=false # ALLUXIO_USER_FILE_PASSIVE_CACHE_ENABLED
# 新增的配置项
# User properties
alluxio.job.worker.threadpool.size=32
alluxio.locality.compare.node.ip=true
alluxio.master.journal.type=UFS
alluxio.master.persistence.blacklist=.staging,_temporary,.sparkStaging
alluxio.underfs.hdfs.configuration=/usr/local/service/hadoop/etc/hadoop/core-site.xml:/usr/local/service/hadoop/etc/hadoop/hdfs-site.xml
alluxio.underfs.version=2.8.5
alluxio.user.client.cache.async.write.threads=16
alluxio.user.metrics.collection.enabled=true
alluxio.user.short.circuit.preferred=true
alluxio.user.streaming.data.timeout=90sec
alluxio.worker.file.buffer.size=5MB
alluxio.worker.network.netty.boss.threads=16
alluxio.worker.network.writer.buffer.size.messages=16
# hadoop-env.sh
HADOOP_CLASSPATH=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar:$HADOOP_CLASSPATH
# hive-env.sh
EXPORT_HIVE_AUX_JARS_PATH=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
# hive-site.xml
spark.driver.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
spark.executor.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
# spark-defaults.conf
spark.driver.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
spark.executor.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
alluxio fsadmin report
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。