Hive搭建

程序员文章站 2022-07-14 14:45:12

...

环境配置

解压安装包

tar -zxvf *hadoop*.gz -C /usr/local/src/
tar -zxvf *spark*.tgz -C /usr/local/src/
tar -zxvf *hive* -C /usr/local/src/
tar -zxvf *jdk* -C /usr/local/src/

修改文件名

mv apache-hive-2.0.0-src/ hive
mv hadoop-2.7.1/ hadoop
mv jdk1.8.0_152/ java
mv spark-3.0.2-bin-hadoop2.7/ spark

配置hosts

vi /etc/hosts

192.168.52.12  master

设置免密

ssh-******

Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FDfzqp1rkSFfB9K8krv86WdV3emYcHSup/nwpMHzHVs [email protected]
The key's randomart image is:
+---[RSA 2048]----+
|        . +o     |
|         o.++. . |
|        .  oooo +|
|       .. +oo..o+|
|        So.*o.= .|
|         o=..+ o.|
|        ..oo =++E|
|          +. [email protected] =|
|         ..o++.=.|
+----[SHA256]-----+

ssh-copy-id master

配置环境变量

vi /etc/profile

export JAVA_HOME=/usr/local/src/java
export PATH=$PATH:$JAVA_HOME/bin
export HADOOP_HOME=/usr/local/src/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export SPARK_HOME=/usr/local/src/spark
export PATH=$PATH:$SPARK_HOME/bin
export HIVE_HOME=/usr/local/src/hive
export PATH=$PATH:$HIVE_HOME/bin

source /etc/profile

Hadoop搭建

配置Hadoop文件

core-site.xml

vi core-site.xml

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://master:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/usr/local/src/tmps/hadoop</value>
        </property>
</configuration>

hdfs-site.xml

vi hdfs-site.xml

<configuration>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/usr/local/src/tmps/hadoop/name</value>
                <description>为了保证元数据的安全一般配置多个不同目录</description>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/usr/local/src/tmps/hadoop/data</value>
                <description>datanode 的数据存储目录</description>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
                <description>HDFS 的数据块的副本存储个数, 默认是3</description>
        </property>
</configuration>

mapred-site.xml

cp mapred-site.xml.template mapred-site.xml
vi mapred-site.xml

<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

yarn-site.xml

vi yarn-site.xml

<configuration>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
                <description>YARN集群为 MapReduce 程序提供的 shuffle 服务</description>
        </property>
</configuration>

hadoop-env.sh

export JAVA_HOME=/usr/local/src/java

初始化Hadoop

hdfs namenode -format

启动Hadoop

start-all.sh

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

Hive搭建

配置MySQL

下载MySQL

rpm -ivh https://repo.mysql.com//mysql57-community-release-el7-11.noarch.rpm
yum install mysql-community-server

开启免密

vi /etc/my.cnf
# 设置跳过密码登录mysql
skip-grant-tables 

default-storage-engine=innodb
innodb_file_per_table
collation-server=utf8_general_ci
init-connect='SET NAMES utf8'
character-set-server=utf8

启动MySQL

systemctl start mysqld.service

查询状态

systemctl status mysqld

进入MySQL

mysql
use mysql
# 刷新环境
flush privileges;

设置密码

set password for [email protected]'localhost' = password('root');

权限设置

# 修改 mysql 库下的 user 表中的 root 用户允许任意 ip 连接
update mysql.user set host='%' where user='root';
# 重新加载数据
flush privileges;

关闭免密

注释掉：skip-grant-tables

配置Hive

配置hive-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- jdbc 连接的 URL -->
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true&amp;characterEncoding=UTF-8&amp;useSSL=false</value>
</property>
<!-- jdbc 连接的 Driver-->
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<!-- jdbc 连接的 username-->
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
</property>
<!-- jdbc 连接的 password -->
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>root</value>
</property>
<!-- Hive 元数据存储版本的验证 -->
<property>
<name>hive.metastore.schema.verification</name>
<value>false</value>
</property>
<!--元数据存储授权-->
<property>
<name>hive.metastore.event.db.notification.api.auth</name>
<value>false</value>
</property>
<!-- Hive 默认在 HDFS 的工作目录 -->
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive/warehouse</value>
</property>
</configuration>

导入jar包

将mysql-connector-java-5.1.38.jar放入$HIVE_HOME/lib

格式化

schematool -initSchema -dbType mysql

环境配置

解压安装包

tar -zxvf *hadoop*.gz -C /usr/local/src/
tar -zxvf *spark*.tgz -C /usr/local/src/
tar -zxvf *hive* -C /usr/local/src/
tar -zxvf *jdk* -C /usr/local/src/

修改文件名

mv apache-hive-2.0.0-src/ hive
mv hadoop-2.7.1/ hadoop
mv jdk1.8.0_152/ java
mv spark-3.0.2-bin-hadoop2.7/ spark

配置hosts

vi /etc/hosts

192.168.52.12  master

设置免密

ssh-******

Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FDfzqp1rkSFfB9K8krv86WdV3emYcHSup/nwpMHzHVs [email protected]
The key's randomart image is:
+---[RSA 2048]----+
|        . +o     |
|         o.++. . |
|        .  oooo +|
|       .. +oo..o+|
|        So.*o.= .|
|         o=..+ o.|
|        ..oo =++E|
|          +. [email protected] =|
|         ..o++.=.|
+----[SHA256]-----+

ssh-copy-id master

配置环境变量

vi /etc/profile

export JAVA_HOME=/usr/local/src/java
export PATH=$PATH:$JAVA_HOME/bin
export HADOOP_HOME=/usr/local/src/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export SPARK_HOME=/usr/local/src/spark
export PATH=$PATH:$SPARK_HOME/bin
export HIVE_HOME=/usr/local/src/hive
export PATH=$PATH:$HIVE_HOME/bin

source /etc/profile

Hadoop搭建

配置Hadoop文件

core-site.xml

vi core-site.xml

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://master:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/usr/local/src/tmps/hadoop</value>
        </property>
</configuration>

hdfs-site.xml

vi hdfs-site.xml

<configuration>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/usr/local/src/tmps/hadoop/name</value>
                <description>为了保证元数据的安全一般配置多个不同目录</description>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/usr/local/src/tmps/hadoop/data</value>
                <description>datanode 的数据存储目录</description>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
                <description>HDFS 的数据块的副本存储个数, 默认是3</description>
        </property>
</configuration>

mapred-site.xml

cp mapred-site.xml.template mapred-site.xml
vi mapred-site.xml

<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

yarn-site.xml

vi yarn-site.xml

<configuration>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
                <description>YARN集群为 MapReduce 程序提供的 shuffle 服务</description>
        </property>
</configuration>

hadoop-env.sh

export JAVA_HOME=/usr/local/src/java

初始化Hadoop

hdfs namenode -format

启动Hadoop

start-all.sh

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

Hive搭建

配置MySQL

下载MySQL

rpm -ivh https://repo.mysql.com//mysql57-community-release-el7-11.noarch.rpm
yum install mysql-community-server

开启免密

vi /etc/my.cnf
# 设置跳过密码登录mysql
skip-grant-tables 

default-storage-engine=innodb
innodb_file_per_table
collation-server=utf8_general_ci
init-connect='SET NAMES utf8'
character-set-server=utf8

启动MySQL

systemctl start mysqld.service

查询状态

systemctl status mysqld

进入MySQL

mysql
use mysql
# 刷新环境
flush privileges;

设置密码

set password for [email protected]'localhost' = password('root');

权限设置

# 修改 mysql 库下的 user 表中的 root 用户允许任意 ip 连接
update mysql.user set host='%' where user='root';
# 重新加载数据
flush privileges;

关闭免密

注释掉：skip-grant-tables

配置Hive

配置hive-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- jdbc 连接的 URL -->
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true&amp;characterEncoding=UTF-8&amp;useSSL=false</value>
</property>
<!-- jdbc 连接的 Driver-->
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<!-- jdbc 连接的 username-->
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
</property>
<!-- jdbc 连接的 password -->
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>root</value>
</property>
<!-- Hive 元数据存储版本的验证 -->
<property>
<name>hive.metastore.schema.verification</name>
<value>false</value>
</property>
<!--元数据存储授权-->
<property>
<name>hive.metastore.event.db.notification.api.auth</name>
<value>false</value>
</property>
<!-- Hive 默认在 HDFS 的工作目录 -->
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive/warehouse</value>
</property>
</configuration>

导入jar包

将mysql-connector-java-5.1.38.jar放入$HIVE_HOME/lib

格式化

schematool -initSchema -dbType mysql

相关标签：大数据 hive mysql hadoop

上一篇： Hive搭建

下一篇： Hive搭建

Hive搭建

环境配置

解压安装包

修改文件名

配置hosts

设置免密

配置环境变量

Hadoop搭建

配置Hadoop文件

core-site.xml

hdfs-site.xml

mapred-site.xml

yarn-site.xml

hadoop-env.sh

初始化Hadoop

启动Hadoop

关闭防火墙

Hive搭建

配置MySQL

下载MySQL

开启免密

启动MySQL

查询状态

进入MySQL

设置密码

权限设置

关闭免密

配置Hive

配置hive-site.xml

导入jar包

格式化

环境配置

解压安装包

修改文件名

配置hosts

设置免密

配置环境变量

Hadoop搭建

配置Hadoop文件

core-site.xml

hdfs-site.xml

mapred-site.xml

yarn-site.xml

hadoop-env.sh

初始化Hadoop

启动Hadoop

关闭防火墙

Hive搭建

配置MySQL

下载MySQL

开启免密

启动MySQL

查询状态

进入MySQL

设置密码

权限设置

关闭免密

配置Hive

配置hive-site.xml

导入jar包

格式化

运用springboot搭建并部署web项目的示例

ActiveMQ基于zookeeper的主从（levelDB Master/Slave）搭建

IOS开发QQ空间/朋友圈类界面的搭建

JAVA开发环境搭建教程

CentOS下搭建PHP环境与WordPress博客程序的全流程总结

springboot2.0+elasticsearch5.5+rabbitmq搭建搜索服务的坑

springmvc+maven搭建web项目

maven搭建spring项目(图文教程)

Flask框架的学习指南之开发环境搭建

Spring Cloud之配置中心的搭建