📔Linux Hadoop 伪分布式安装(Hive on Spark)
版本概要:
- jdk: jdk-8u391-linux-x64.tar.gz
- hadoop:hadoop-3.3.1.tar.gz
- hive:apache-hive-3.1.2-bin.tar.g
- mysql:mysql-8.0.27-1.el7.x86_64.rpm-bundle.tar
- maven:apache-maven-3.5.4-bin.tar.gz
- scala:scala-2.11.12.tgz
- spark:spark-2.3.0.tgz
- zookeeper:zookeeper-3.4.10.tar.gz
- hbase:hbase-2.4.12-bin.tar.gz
# 解压
[root@sole install]# tar -zxvf jdk-8u391-linux-x64.tar.gz -C /opt/software/
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
# JAVA_HOME
export JAVA_HOME=/opt/software/jdk1.8.0_391
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib:$CLASSPATH
export JAVA_PATH=${JAVA_HOME}/bin:${JRE_HOME}/bin
export PATH=$PATH:${JAVA_PATH}
# 重新加载
[root@sole ~]# source /etc/profile
# 解压
[root@sole install]# tar -zxvf hadoop-3.3.1.tar.gz -C /opt/software/
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
# HADOOP_HOME
export HADOOP_HOME=/opt/software/hadoop-3.3.1
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
# 重新加载
[root@sole ~]# source /etc/profile
# Hadoop配置修改
[root@sole software]# cd hadoop-3.3.1/etc/hadoop/
[root@sole hadoop]# vi hadoop-env.sh
export JAVA_HOME=/opt/software/jdk1.8.0_391
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
[root@sole hadoop]# vi core-site.xml
<configuration>
<property>
<name>fs.defaultFSname>
<value>hdfs://sole:9000value>
property>
<property>
<name>hadoop.tmp.dirname>
<value>/opt/software/hadoop-3.3.1/hadoopdatavalue>
property>
configuration>
[root@sole hadoop]# vi hdfs-site.xml
<configuration>
<property>
<name>dfs.replicationname>
<value>1value>
property>
<property>
<name>dfs.namenode.name.dirname>
<value>/opt/software/hadoop-3.3.1/tmp/namevalue>
property>
<property>
<name>dfs.datanode.data.dirname>
<value>/opt/software/hadoop-3.3.1/tmp/datavalue>
property>
configuration>
[root@sole hadoop]# vi yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-servicesname>
<value>mapreduce_shufflevalue>
property>
<property>
<name>yarn.resourcemanager.addressname>
<value>sole:18040value>
property>
<property>
<name>yarn.resourcemanager.scheduler.addressname>
<value>sole:18030value>
property>
<property>
<name>yarn.resourcemanager.resource-tracker.addressname>
<value>sole:18025value>
property>property>
<property>
<name>yarn.resourcemanager.admin.addressname>
<value>sole:18141value>
property>
<property>
<name>yarn.resourcemanager.webapp.addressname>
<value>sole:18088value>
property>
configuration>
[root@sole hadoop]# vi mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.namename>
<value>yarnvalue>
property>
configuration>
# 初始化,进到${HADOOP_HOME}/sbin目录下
[root@sole sbin]# pwd
/opt/software/hadoop-3.3.1/sbin
[root@sole sbin]# hdfs namenode -format
# 启动服务
[root@sole sbin]# start-dfs.sh
[root@sole sbin]# start-yarn.sh
# 卸载已有mariadb服务,如果已安装过MySQL,则将旧的MySQL服务全部卸载再安装
[root@sole ~]# rpm -qa|grep mariadb
[root@sole ~]# yum remove mariadb-libs
[root@sole ~]# tar -xvf mysql-8.0.27-1.el7.x86_64.rpm-bundle.tar
# 为了避免安装过程中报错,提前安装好以下依赖
[root@sole ~]# -y install libaio
[root@sole ~]# yum install openssl-devel.x86_64 openssl.x86_64 -y
[root@sole ~]# yum -y install autoconf
[root@sole ~]# yum install perl.x86_64 perl-devel.x86_64 -y
[root@sole ~]# yum install perl-JSON.noarch -y
[root@sole ~]# yum install perl-Test-Simple
[root@sole ~]# yum install net-tools
# 安装mysql
[root@sole ~]# rpm -ivh mysql-community-common-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rpm -ivh mysql-community-client-plugins-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rmp -ivh mysql-community-libs-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rpm -ivh mysql-community-client-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rpm -ivh mysql-community-server-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rpm -ivh mysql-community-libs-compat-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rpm -ivh mysql-community-embedded-compat-8.0.27-1.el7.x86_64.rpm
[root@sole ~]# rpm -ivh mysql-community-devel-8.0.27-1.el7.x86_64.rpm
#启动MySQL
[root@sole ~]# mysqld --initialize --console
[root@sole ~]# chown -R mysql:mysql /var/lib/mysql/
[root@sole ~]# systemctl start mysqld.service
[root@sole ~]# systemctl status mysqld.service
[root@sole ~]# cat /var/log/mysqld.log |grep password --查看临时密码
# 修改密码&远程登录权限
mysql> alter user 'root'@'localhost' identified by 'root';
mysql> CREATE USER 'root'@'%' IDENTIFIED BY 'root';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION;
mysql> FLUSH PRIVILEGES;
# MySQL字符集修改,最后添加配置如下
[root@sole ~]# vi /etc/my.cnf
[mysql.server]
default-character-set = utf8
[client]
default-character-set = utf8
#添加完重启服务
[root@sole ~]# service mysqld restart
# 解压
[root@sole install]# tar -zxvf apache-hive-3.1.2-bin.tar.gz -C /opt/software/
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
# HIVE_HOME
export HIVE_HOME=/opt/software/apache-hive-3.1.2-bin
export PATH=$HIVE_HOME/bin:$PATH
# 重新加载
[root@sole ~]# source /etc/profile
# Hive配置修改,进入${HIVE_HOME}/conf
[root@sole conf]# cp hive-env.sh.template hive-env.sh
[root@sole conf]# vi hive-env.sh
export JAVA_HOME=/opt/software/jdk1.8.0_391
export HADOOP_HOME=/opt/software/hadoop-3.3.1
export HIVE_CONF_DIR=/opt/software/apache-hive-3.1.2-bin/conf
export HIVE_AUX_JARS_PATH=/opt/software/apache-hive-3.1.2-bin/lib
[root@sole conf]# vi hive-site.xml
<configuration>
<property>
<name>hive.metastore.warehouse.dirname>
<value>/user/hive/warehousevalue>
property>
<property>
<name>javax.jdo.option.ConnectionURLname>
<value>jdbc:mysql://sole:3306/hive?createDatabaseIfNotExist=true&useSSL=false&useUnicode=true&characterEncoding=UTF-8&allowPublicKeyRetrieval=truevalue>
property>
<property>
<name>javax.jdo.option.ConnectionDriverNamename>
<value>com.mysql.cj.jdbc.Drivervalue>
property>
<property>
<name>javax.jdo.option.ConnectionUserNamename>
<value>rootvalue>
property>
<property>
<name>javax.jdo.option.ConnectionPasswordname>
<value>rootvalue>
property>
<property>
<name>hive.metastore.schema.verificationname>
<value>falsevalue>
property>
<property>
<name>system:user.namename>
<value>rootvalue>
<description>user namedescription>
property>
<property>
<name>hive.server2.thrift.bind.hostname>
<value>solevalue>
<description>Bind host on which to run the HiveServer2 Thrift service.description>
property>
<property>
<name>hive.server2.thrift.portname>
<value>11000value>
property>
<property>
<name>hive.metastore.urisname>
<value>thrift://sole:9083value>
property>
<property>
<name>spark.yarn.jarsname>
<value>hdfs:///spark/spark-2.3.0-jars/*.jarvalue>
property>
<property>
<name>hive.execution.enginename>
<value>sparkvalue>
property>
<property>
<name>hive.spark.client.connect.timeoutname>
<value>100000msvalue>
property>
configuration>
# MySQL驱动
[root@sole install]# cp mysql-connector-j-8.0.33.jar /opt/software/apache-hive-3.1.2-bin/lib/
<dependency> <groupId>com.mysqlgroupId> <artifactId>mysql-connector-jartifactId> <version>8.0.33version> dependency>
- 1
- 2
- 3
- 4
- 5
# 解决Hadoop和Hive的两个guava.jar版本冲突问题:
# 删除${HIVE_HOME}/lib中的guava-19.0.jar
# 并将${HADOOP_HOME}/share/hadoop/common/lib/guava-27.0-jre.jar复制到${HIVE_HOME}/lib下
[root@sole install]# cd /opt/software/apache-hive-3.1.2-bin/lib/
[root@sole lib]# rm -f guava-19.0.jar
[root@sole lib]# cp /opt/software/hadoop-3.3.1/share/hadoop/common/lib/guava-27.0-jre.jar ./
# 初始化元数据库
[root@sole bin]# ./schematool -dbType mysql -initSchema
# 启动服务 metastore & hiveserver2
[root@sole bin]# nohup hive --service metastore>hive.log 2>&1 &
[root@sole bin]# nohup hive --service hiveserver2>/dev/null 2>&1 &
# 解压
[root@sole install]# tar -zxvf apache-maven-3.5.4-bin.tar.gz -C /opt/software
[root@sole software]# mv apache-maven-3.5.4/ maven-3.5.4
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
# MAVEN_HOME
export MAVEN_HOME=/opt/software/maven-3.5.4
export PATH=$MAVEN_HOME/bin:$PATH
# 重新加载环境变量并测试maven
[root@sole ~]# source /etc/profile
[root@sole ~]# mvn -v
# 配置阿里云镜像
[root@sole software]# vi maven-3.5.4/conf/settings.xml
<mirror>
<id>alimavenid>
<name>aliyun mavenname>
<url>http://maven.aliyun.com/nexus/content/groups/public/url>
<mirrorOf>centralmirrorOf>
mirror>
# 解压
[root@sole install]# tar -zxvf scala-2.11.12.tgz -C /opt/software
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
#SCALA_HOME
export SCALA_HOME=/opt/software/scala-2.11.12
export PATH=$SCALA_HOME/bin:$PATH
# 重新加载并测试
[root@sole ~]# source /etc/profile
[root@sole ~]# scala -version
# 解压
[root@sole install]# tar -zxvf spark-2.3.0.tgz -C /opt/software
# 编译
[root@sole software]# cd spark-2.3.0/
[root@sole spark-2.3.0]# ./dev/make-distribution.sh --name without-hive --tgz -Pyarn -Phadoop-2.7 -Dhadoop.version=3.3.1 -Pparquet-provided -Porc-provided -Phadoop-provided
# 解压
[root@sole spark-2.3.0]# tar -zxvf spark-2.3.0-bin-without-hive.tgz -C /opt/software/
# 配置
[root@sole spark--bin-build]# vi /etc/profile.d/my.sh
# SPARK_HOME
export SPARK_HOME=/opt/software/spark-2.3.0-bin-without-hive
export SPARK_CLASSPATH=$SPARK_HOME/jars
export PATH=$SPARK_HOME/bin:$PATH
##########################################################################################
[root@sole spark-2.3.0-bin-without-hive]# cd conf/
[root@sole conf]# cp spark-defaults.conf.template spark-defaults.conf
[root@sole conf]# vi spark-defaults.conf
spark.master yarn
spark.home /opt/software/spark-2.3.0-bin-without-hive
spark.eventLog.enabled true
spark.eventLog.dir hdfs://sole/tmp/spark
spark.serializer org.apache.spark.serializer.KryoSerializer
spark.executor.memory 1g
spark.driver.memory 1g
spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
spark.yarn.archive hdfs:///spark/jars/spark2.3.0-without-hive-libs.jar
spark.yarn.jars hdfs:///spark/jars/spark2.3.0-without-hive-libs.jar
##########################################################################################
[root@sole conf]# cp spark-env.sh.template spark-env.sh
[root@sole conf]# vi spark-env.sh
export SPARK_DIST_CLASSPATH=$(hadoop classpath)
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop/
##########################################################################################
# jar包上传替换
[root@sole spark-2.3.0-bin-without-hive]# hadoop fs -mkdir -p hdfs://sole:8082/tmp/spark
[root@sole spark-2.3.0-bin-without-hive]# hadoop fs -mkdir -p /spark/spark-2.3.0-jars
[root@sole spark-2.3.0-bin-without-hive]# jar cv0f spark2.3.0-without-hive-libs.jar -C ./jars/ .
[root@sole spark-2.3.0-bin-without-hive]# hadoop fs -mkdir -p /spark/jars
[root@sole spark-2.3.0-bin-without-hive]# hadoop fs -put spark2.3.0-without-hive-libs.jar /spark/jars/
[root@sole spark-2.3.0-bin-without-hive]# hadoop fs -put ./jars/* /spark/spark-2.3.0-jars/
[root@sole spark-2.3.0-bin-without-hive]# cp jars/scala-library-2.11.8.jar ../apache-hive-3.1.2-bin/lib/
[root@sole spark-2.3.0-bin-without-hive]# cp jars/spark-core_2.11-2.3.0.jar ../apache-hive-3.1.2-bin/lib/
[root@sole spark-2.3.0-bin-without-hive]# cp jars/spark-network-common_2.11-2.3.0.jar ../apache-hive-3.1.2-bin/lib/
# 启动hive服务测试
[root@sole ~]# nohup hive --service metastore>hive.log 2>&1 &
[root@sole ~]# nohup hive --service hiveserver2>/dev/null 2>&1 &
[root@sole ~]# hive
# 解压
[root@sole install]# tar -zxvf zookeeper-3.4.10.tar.gz -C /opt/software/
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
# ZK_HOME
export ZK_HOME=/opt/software/zookeeper-3.4.10
export PATH=$ZK_HOME/bin:$ZK_HOME/sbin:$PATH
# 重新加载
[root@sole ~]# source /etc/profile
# 配置
[root@sole install]# cd /opt/software/zookeeper-3.4.10/
# 在zookeeper的根目录下新建文件夹mydata
[root@sole zookeeper-3.4.10]# mkdir mydata
[root@sole zookeeper-3.4.10]# touch myid
[root@sole zookeeper-3.4.10]# echo "1" >> myid
##########################################################################################
[root@sole zookeeper-3.4.10]# cd /opt/software/zookeeper-3.4.10/conf
[root@sole conf]# cp zoo_sample.cfg zoo.cfg
[root@sole conf]# vi zoo.cfg
# 在zoo.cfg这个文件中,配置集群信息是存在一定的格式:service.N =YYY:A:B
# N:代表服务器编号(也就是myid里面的值);
# YYY:服务器地址/hostname;
# A:表示 Flower 跟 Leader的通信端口,简称服务端内部通信的端口(默认2888);
# B:表示 是选举端口(默认是3888);
dataDir=/opt/software/zookeeper-3.4.10/mydata
server.1=sole:2888:3888
# 启动服务
[root@sole ~]# zkServer.sh start
# 解压
[root@sole install]# tar -zxvf hbase-2.4.12-bin.tar.gz -C /opt/software/
##########################################################################################
# 编辑环境变量
[root@sole ~]# vi /etc/profile.d/my.sh
#HBASE_HOME
export HBASE_HOME=/opt/software/hbase-2.4.12
export PATH=$HBASE_HOME/bin:$PATH
# 重新加载
[root@sole ~]# source /etc/profile
# 配置
[root@sole ~]# cd /opt/software/hbase-2.4.12/conf/
[root@sole conf]# vi hbase-env.sh
export JAVA_HOME=/opt/software/jdk1.8.0_391
[root@sole conf]# vi hbase-site.xml
<configuration>
<property>
<name>hbase.rootdirname>
<value>hdfs://sole:9000/hbasevalue>
property>
<property>
<name>hbase.cluster.distributedname>
<value>truevalue>
property>
<property>
<name>hbase.tmp.dirname>
<value>./tmpvalue>
property>
<property>
<name>hbase.zookeeper.property.clientPortname>
<value>2181value>
property>
<property>
<name>hbase.zookeeper.property.dataDirname>
<value>/opt/software/zookeeper-3.4.10/mydatavalue>
property>
<property>
<name>hbase.unsafe.stream.capability.enforcename>
<value>falsevalue>
property>
configuration>
# 启动服务
[root@sole ~]# start-hbase.sh
PS:如果有写错或者写的不好的地方,欢迎各位大佬在评论区留下宝贵的意见或者建议,敬上!如果这篇博客对您有帮助,希望您可以顺手帮我点个赞!不胜感谢!
原创作者:wsjslient |