大数据与应用服务赛题复现-JDK 与 Hadoop 安装配置

SOFT=/opt/downloads; LOCAL=/usr/local
sudo -n tar -zxf $SOFT/jdk-8u202-linux-x64.tar.gz -C $LOCAL/
sudo -n mv $LOCAL/jdk1.8.0_202 $LOCAL/jdk
sudo -n tar -zxf $SOFT/hadoop-3.3.4.tar.gz -C $LOCAL/
sudo -n mv $LOCAL/hadoop-3.3.4 $LOCAL/hadoop
sudo -n mkdir -p $LOCAL/data/tmp $LOCAL/data/name $LOCAL/data/data
sudo -n chown -R dubhe:dubhe $LOCAL/jdk $LOCAL/hadoop $LOCAL/data

（无输出即成功）

ls -d /usr/local/jdk /usr/local/hadoop /usr/local/data

sudo -n tee /etc/profile.d/00-java.sh >/dev/null <<'EOF'
export JAVA_HOME=/usr/local/jdk
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib
export PATH=$JAVA_HOME/bin:$PATH
EOF

sudo -n tee /etc/profile.d/hadoop.sh >/dev/null <<'EOF'
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
EOF

sudo -n chmod +x /etc/profile.d/00-java.sh /etc/profile.d/hadoop.sh
source /etc/profile
java -version 2>&1 | head -1
hadoop version | head -1

java version "1.8.0_202"
Hadoop 3.3.4

echo $JAVA_HOME   # /usr/local/jdk
which java        # /usr/local/jdk/bin/java

CONF=/usr/local/hadoop/etc/hadoop

# (1) hadoop-env.sh：JAVA_HOME + 五个 *_USER

sudo -n sed -i 's|^#*\s*export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk|' $CONF/hadoop-env.sh

for kv in HDFS_NAMENODE_USER HDFS_DATANODE_USER HDFS_SECONDARYNAMENODE_USER \

          YARN_RESOURCEMANAGER_USER YARN_NODEMANAGER_USER; do

  grep -q "export $kv" $CONF/hadoop-env.sh || echo "export $kv=dubhe" | sudo -n tee -a $CONF/hadoop-env.sh >/dev/null

done

# (2) core-site.xml

sudo -n tee $CONF/core-site.xml >/dev/null <<'EOF'

<?xml version="1.0" encoding="UTF-8"?>

<configuration>

  <property><name>fs.defaultFS</name><value>hdfs://master:9000</value></property>

  <property><name>hadoop.tmp.dir</name><value>/usr/local/data/tmp</value></property>

  <property><name>hadoop.proxyuser.dubhe.hosts</name><value>*</value></property>

  <property><name>hadoop.proxyuser.dubhe.groups</name><value>*</value></property>

</configuration>

EOF

# (3) hdfs-site.xml（★ rpc/http-bind-host 必写，否则 Ubuntu 默认的

#     127.0.1.1 会把 NameNode 绑到回环，从节点连不上）

sudo -n tee $CONF/hdfs-site.xml >/dev/null <<'EOF'

<?xml version="1.0" encoding="UTF-8"?>

<configuration>

  <property><name>dfs.replication</name><value>3</value></property>

  <property><name>dfs.namenode.name.dir</name><value>file:///usr/local/data/name</value></property>

  <property><name>dfs.datanode.data.dir</name><value>file:///usr/local/data/data</value></property>

  <property><name>dfs.namenode.http-address</name><value>master:9870</value></property>

  <property><name>dfs.namenode.secondary.http-address</name><value>master:50090</value></property>

  <property><name>dfs.permissions.enabled</name><value>false</value></property>

  <property><name>dfs.namenode.datanode.registration.ip-hostname-check</name><value>false</value></property>

  <property><name>dfs.namenode.rpc-bind-host</name><value>0.0.0.0</value></property>

  <property><name>dfs.namenode.http-bind-host</name><value>0.0.0.0</value></property>

</configuration>

EOF

# (4) mapred-site.xml

sudo -n tee $CONF/mapred-site.xml >/dev/null <<'EOF'

<?xml version="1.0" encoding="UTF-8"?>

<configuration>

  <property><name>mapreduce.framework.name</name><value>yarn</value></property>

  <property><name>yarn.app.mapreduce.am.env</name><value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value></property>

  <property><name>mapreduce.map.env</name><value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value></property>

  <property><name>mapreduce.reduce.env</name><value>HADOOP_MAPRED_HOME=/usr/local/hadoop</value></property>

  <property><name>mapreduce.jobhistory.webapp.address</name><value>master:19888</value></property>

</configuration>

EOF

# (5) yarn-site.xml（★ bind-host 同上，ResourceManager 也受 127.0.1.1 影响）

sudo -n tee $CONF/yarn-site.xml >/dev/null <<'EOF'

<?xml version="1.0" encoding="UTF-8"?>

<configuration>

  <property><name>yarn.resourcemanager.hostname</name><value>master</value></property>

  <property><name>yarn.resourcemanager.bind-host</name><value>0.0.0.0</value></property>

  <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property>

  <property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property>

  <property><name>yarn.nodemanager.resource.memory-mb</name><value>2048</value></property>

  <property><name>yarn.scheduler.minimum-allocation-mb</name><value>512</value></property>

  <property><name>yarn.scheduler.maximum-allocation-mb</name><value>2048</value></property>

  <property><name>yarn.nodemanager.resource.cpu-vcores</name><value>2</value></property>

  <property><name>yarn.nodemanager.vmem-check-enabled</name><value>false</value></property>

  <property><name>yarn.log-aggregation-enable</name><value>true</value></property>

</configuration>

EOF

# (6) workers（Hadoop 3.x 只认 workers，不认 slaves）

printf "master\nslave1\nslave2\n" | sudo -n tee $CONF/workers >/dev/null

sudo -n chown -R dubhe:dubhe /usr/local/hadoop

（无输出即成功）

cat /usr/local/hadoop/etc/hadoop/workers
grep JAVA_HOME /usr/local/hadoop/etc/hadoop/hadoop-env.sh | tail -1

# 打包（比 rsync 目录更稳，避免 /usr/local 权限问题）
sudo -n tar -zcf /tmp/jdk.tgz -C /usr/local jdk
sudo -n tar -zcf /tmp/hadoop.tgz -C /usr/local hadoop
sudo -n chown dubhe:dubhe /tmp/jdk.tgz /tmp/hadoop.tgz

for h in slave1 slave2; do
  rsync -a --partial /tmp/jdk.tgz /tmp/hadoop.tgz dubhe@$h:/tmp/
  scp -q /etc/profile.d/00-java.sh /etc/profile.d/hadoop.sh dubhe@$h:/tmp/
  ssh $h 'cat > /tmp/deploy.sh <<"EOS"
#!/bin/bash
rm -rf /usr/local/jdk /usr/local/hadoop
tar -zxf /tmp/jdk.tgz -C /usr/local/
tar -zxf /tmp/hadoop.tgz -C /usr/local/
mv -f /tmp/00-java.sh /etc/profile.d/00-java.sh
mv -f /tmp/hadoop.sh  /etc/profile.d/hadoop.sh
chmod +x /etc/profile.d/00-java.sh /etc/profile.d/hadoop.sh
chown -R dubhe:dubhe /usr/local/jdk /usr/local/hadoop
mkdir -p /usr/local/data/tmp /usr/local/data/name /usr/local/data/data
chown -R dubhe:dubhe /usr/local/data
echo DEPLOY_OK
EOS
chmod +x /tmp/deploy.sh'
  ssh $h 'echo Test123 | sudo -S /tmp/deploy.sh'
done

DEPLOY_OK
DEPLOY_OK

for h in master slave1 slave2; do
  ssh $h 'source /etc/profile; printf "%-8s %s | %s\n" $(hostname) "$(java -version 2>&1|head -1)" "$(hadoop version|head -1)"'
done

source /etc/profile
rm -rf /usr/local/data/tmp/* /usr/local/data/name/*
hdfs namenode -format -nonInteractive 2>&1 | tail -5

Storage directory /usr/local/data/name has been successfully formatted.
SHUTDOWN_MSG: Shutting down NameNode at master/172.20.0.100

grep clusterID /usr/local/data/name/current/VERSION

start-dfs.sh 2>&1 | tail -5

start-yarn.sh 2>&1 | tail -5

sleep 10

# ★ 关键自检：9000/8032 必须监听 0.0.0.0

#   出现 127.0.1.1:9000 说明 /etc/hosts 里主机名映射写错（见第 6 章症状 S）

ss -lntp | grep -E ':9000|:8032'

jps

for h in slave1 slave2; do echo "--- $h"; ssh -n $h "source /etc/profile; jps"; done

# 从节点真正注册成功的判据

hdfs dfsadmin -report 2>/dev/null | grep -c "^Name:"   # 期望 3

yarn node -list 2>/dev/null | grep -c RUNNING          # 期望 3

Starting namenodes on [master]
Starting datanodes
Starting secondary namenodes [master]
Starting resourcemanager
Starting nodemanagers

1337 NameNode
1473 DataNode
1699 SecondaryNameNode
1885 ResourceManager
2016 NodeManager
2707 Jps

# 从节点应为 DataNode + NodeManager
--- slave1
5615 DataNode
5755 NodeManager

hdfs dfs -mkdir -p /user/dubhe /tmp/flume /tmp/data/{raw,clean,label,out} /user/hive/warehouse
hdfs dfs -chmod -R 777 /tmp /user/hive
echo "hello bigdata" > /tmp/smoke.txt
hdfs dfs -put -f /tmp/smoke.txt /user/dubhe/
hdfs dfs -cat /user/dubhe/smoke.txt
hdfs dfsadmin -report 2>/dev/null | grep -c "^Name:" | xargs -I{} echo "Live DataNodes: {}"
yarn node -list 2>/dev/null | tail -1

hello bigdata
Live DataNodes: 3

curl -s -o /dev/null -w "9870=%{http_code} " http://172.20.0.100:9870
curl -s -o /dev/null -w "8088=%{http_code}\n" http://172.20.0.100:8088
# 9870=302 8088=302   ← 302 是正常跳转