1. 程式人生 > >hadoop生態搭建(3節點)-07.hive配置

hadoop生態搭建(3節點)-07.hive配置

所有 chm path yarn 啟動 find extern ppc 修改

# http://archive.apache.org/dist/hive/hive-2.1.1/

# ==================================================================安裝 hive

tar -zxvf apache-hive-2.1.1-bin.tar.gz -C /usr/local
mv /usr/local/apache-hive-2.1.1-bin /usr/local/hive-2.1.1
rm –r ~/apache-hive-2.1.1-bin.tar.gz


cp ~/mysql-connector-java-5.1.46.jar /usr/local/hive-2.1.1/lib/

# 配置環境變量
# ==================================================================node1 node2 node3

vi /etc/profile

# 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加
export JAVA_HOME=/usr/java/jdk1.8.0_111
export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.6
export MYSQL_HOME=/usr/local/mysql
export HBASE_HOME=/usr/local/hbase-1.2.4
export HIVE_HOME=/usr/local/hive-2.1.1

export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$MYSQL_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

# ==================================================================node1

# 使環境變量生效
source /etc/profile

# 查看配置結果
echo $HIVE_HOME

mkdir -p $HIVE_HOME/data/hive
mkdir -p $HIVE_HOME/data/hive/operaion_logs
mkdir -p $HIVE_HOME/data/resources


# 配置 hive
cp $HIVE_HOME/conf/hive-env.sh.template $HIVE_HOME/conf/hive-env.sh
cp $HIVE_HOME/conf/hive-default.xml.template $HIVE_HOME/conf/hive-site.xml
cp $HIVE_HOME/conf/hive-exec-log4j2.properties.template $HIVE_HOME/conf/hive-exec-log4j2.properties
cp $HIVE_HOME/conf/hive-log4j2.properties.template $HIVE_HOME/conf/hive-log4j2.properties
# ${system:java.io.tmpdir}/${system:user.name} 替換為本機路徑 /usr/local/hive-2.1.1/data/hive
# ${system:java.io.tmpdir}/${hive.session.id}_resources替換為本機路徑 /usr/local/hive-2.1.1/data/resources
# ${system:java.io.tmpdir}/${system:user.name}/operation_logs 替換為本機路徑 /usr/local/hive-2.1.1/data/hive/operation_logs

vi $HIVE_HOME/conf/hive-site.xml

# esc 後輸入
:%s#${system:java.io.tmpdir}/${system:user.name}#/usr/local/hive-2.1.1/data/hive#

:%s#${system:java.io.tmpdir}/${hive.session.id}_resources#/usr/local/hive-2.1.1/data/resources#

# esc / 輸入 hive.exec.scratchdir 找到後<value> 點擊 Insert 鍵後進行修改

<property>
    <name>hive.exec.scratchdir</name>
    <value>/hive/tmp</value>
</property>

<property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/hive/warehouse</value>
</property>

<!-- 通過jdbc協議連接mysql的hive庫 -->
<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://node1:3306/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
</property>

<!-- jdbc的mysql驅動 -->
<property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
</property>

<!-- mysql用戶名 -->
<property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>hive</value>
</property>

<!-- mysql用戶密碼 -->
<property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>Hive-123</value>
</property>

<!--hiveserver2的HA-->
<property>
    <name>hive.server2.support.dynamic.service.discovery</name>
    <value>true</value>
</property>

<property>
    <name>hive.zookeeper.quorum</name>
    <value>node1:2181,node2:2181,node3:2181</value>
</property>

<!-- hive的web頁面暫時不配置 -->
<property>
    <name>hive.hwi.war.file</name>
    <value>/usr/local/hive-2.1.1/lib/hive-hwi-2.1.1.jar</value>
</property>
# 從hbase/lib下復制必要jar包到hive/lib下 
cp $HBASE_HOME/lib/hbase-client-1.2.4.jar $HBASE_HOME/lib/hbase-common-1.2.4.jar $HIVE_HOME/lib


# 同步hive和hadoop的jline版本 
cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib


# 查看版本
# cd $HADOOP_HOME/share/hadoop/yarn/lib
# find ./ -name "*jline*jar"

# 刪除低版本的jline 0.9
# rm jline-0.9.94.jar


# 復制jdk的tools.jar到hive/lib下 
cp $JAVA_HOME/lib/tools.jar $HIVE_HOME/lib

# rm -f $HIVE_HOME/lib/log4j-slf4j-impl-2.4.1.jar


vi $HIVE_HOME/conf/hive-env.sh

HADOOP_HOME=/usr/local/hadoop-2.7.6
export HIVE_HOME=/usr/local/hive-2.1.1
export HIVE_CONF_DIR=/usr/local/hive-2.1.1/conf
export HIVE_AUX_JARS_PATH=/usr/local/hive-2.1.1/lib

# 如果hadoop之前沒有配置hadoop.proxyuser.root.groups需配置

# ==================================================================node1 node2 node3
# 如果沒有權限,在Hadoop的core-site.xml中增加配置:
vi $HADOOP_HOME/etc/hadoop/core-site.xml

<property>
	<name>hadoop.proxyuser.root.groups</name>
	<value>*</value>
</property>
<property>
	<name>hadoop.proxyuser.root.hosts</name>
	<value>*</value>
</property>


# ==================================================================node1
# 使用超級用戶刷新配置
yarn rmadmin -refreshSuperUserGroupsConfiguration
hdfs dfsadmin -refreshSuperUserGroupsConfiguration


# ==================================================================node1 node2
# 如果是對namenode做過HA,則需要在主備namenode上執行
hdfs dfsadmin -fs hdfs://appcluster -refreshSuperUserGroupsConfiguration

# ==================================================================node2 node3

# 使環境變量生效
source /etc/profile

# 查看配置結果
echo $HIVE_HOME

# ==================================================================node1

$HIVE_HOME/bin/schematool -initSchema -dbType mysql


scp -r $HIVE_HOME node2:/usr/local/
scp -r $HIVE_HOME node3:/usr/local/

# 啟動

# ==================================================================node1 node2 node3
# 啟動 zookeeper
zkServer.sh start
zkServer.sh status


# ==================================================================node1
# 啟動hadoop所有進程
$HADOOP_HOME/sbin/start-all.sh


$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc


# ==================================================================node2
$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager
$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc


# ==================================================================node1
# 要實現Hbase的HA(High Availablity)
$HBASE_HOME/bin/hbase-daemon.sh start master

# 啟動Hbase(start-hbase.sh)
$HBASE_HOME/bin/start-hbase.sh


# ==================================================================node2
# 開啟Hbase的HA
$HBASE_HOME/bin/hbase-daemon.sh start master


# ==================================================================node1
$HIVE_HOME/bin/hiveserver2


# ==================================================================node2
$HIVE_HOME/bin/hiveserver2


# ==================================================================node1
zkCli.sh

ls /hiveserver2

get /hiveserver2/serverUri=node1:10000;version=2.1.1;sequence=0000000000


$HIVE_HOME/bin/beeline -u "jdbc:hive2://node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2" root 123456

# $HIVE_HOME/bin/beeline
# > !connect jdbc:hive2://node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2 root "123456"


> create external table user_info(user_id int comment ‘userID‘,user_name string comment ‘userName‘)row format delimited fields terminated by ‘\t‘ lines terminated by ‘\n‘;

> show tables;

mkdir /root/hive

vi /root/hive/user_info.txt

1001	zhangsan
1002	lisi
1003	wangwu


> load data local inpath ‘/root/hive/user_info.txt‘ into table user_info;

> select * from user_info;

> quit;


hdfs dfs -ls /

hdfs dfs -ls /hive/warehouse


hdfs dfs -cat /hive/warehouse/user_info/user_info.txt


hadoop fs -mkdir /hive_input_data


vi /root/hive/user_info.txt

1001	zhangsan
1002	lisi
1003	wangwu
1004	liuliu
1005	qiqi


hadoop fs -put /root/hive/user_info.txt /hive_input_data


hdfs dfs -ls /hive_input_data

# hdfs -dfs  -chmod 777 /hive_input_data

> select * from user_info;

> load data inpath ‘/hive_input_data/user_info.txt‘ overwrite into table user_info;

> select * from user_info;


# ==================================================================node1
# stop已經啟動的進程
$HBASE_HOME/bin/stop-hbase.sh


$HADOOP_HOME/sbin/stop-all.sh


# ==================================================================node1 node2 node3
# 停止 zookeeper
zkServer.sh stop


# ==================================================================node1
$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc


# ==================================================================node2
$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager
$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc



shutdown -h now
# 快照 hive_hiveserver2集群

hadoop生態搭建(3節點)-07.hive配置