hive 和 spark版本之前有強對應關係

apache hive 和 spark 對應關係表

cdh hive 和 spark對應關係

編譯環境準備

下載scala 2.11版本

下載地址

# 新增環境變數
vim /etc/profile
export SCALA_HOME=/root/scala-2.11
 
.12
export PATH=$PATH:$SCALA_HOME/bin

下載maven (3.3 版本以上)

下載地址

# 新增環境變數
vim /etc/profile
export MAVEN_HOME=/root/apache-maven-3.5.3
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -XX:MaxPermSize=2014M"
export PATH=$PATH:$MAVEN_HOME/bin

source /etc/profile

下載原始碼，進行編譯

下載地址

檢視hadoop version

指定hadoop version 編譯

# 確定版本 + 不編譯hive包
./make-distribution.sh --name hadoop2-without-hive --tgz -Pyarn -Phadoop-provided -Phadoop-2.6 -Porc-provided -Dhadoop.version=2.6.0-cdh5.14.2

編譯生成spark-1.6.0-bin-hadoop2-without-hive.tgz
解壓spark-1.6.0-bin-hadoop2-without-hive.tgz 到目錄(eg. /root/spark-1.6.0-bin-hadoop2-without-hive)

新增spark配置檔案

• spark hdfs

sudo -u hdfs hdfs dfs -mkdir -p /spark/jars
sudo -u hdfs hdfs dfs -mkdir -p /spark/log/envent-log
# 上傳 jar包
hdfs dfs -put /root/spark-1.6.0-bin-hadoop2-without-hive/lib/spark-assembly-1.6.0-cdh5.14.2-hadoop2.6.0-cdh5.14.2.jar /user/root
sudo -u hdfs hdfs dfs -mv /user/root/spark-assembly-1.6.0-cdh5.14.2-hadoop2.6.0-cdh5.14.2.jar /spark/jars
sudo -u hdfs hdfs dfs -chown hdfs /spark/jars/spark-assembly-1.6.0-cdh5.14.2-hadoop2.6.0-cdh5.14.2.jar
sudo -u hdfs hdfs dfs -chmod 777 /spark/jars/spark-assembly-1.6.0-cdh5.14.2-hadoop2.6.0-cdh5.14.2.jar

• spark-evn.sh

vim spark-1.6.0-bin-hadoop2-without-hive/conf/spark-env.sh
export JAVA_HOME=/usr/java/default
export SPARK_HOME=/root/spark-1.6.0-bin-hadoop2-without-hive
export HADOOP_HOME=/usr/lib/hadoop
export HADOOP_CONF_DIR=/etc/hadoop/conf
export YARN_CONF_DIR=/etc/hadoop/conf
export SPARK_LIBARY_PATH=$SPARK_LIBARY_PATH:$HADOOP_HOME/lib/native
export SPARK_CLASSPATH=$SPARK_CLASSPATH:$HADOOP_HOME/lib/*
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/
export SPARK_DIST_CLASSPATH=$(hadoop classpath)
export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=17777 -Dspark.history.fs.logDirectory=hdfs://xiwu-cluster/spark/log/envent-log"

• spark-defaults.conf

vim spark-1.6.0-bin-hadoop2-without-hive/conf/spark-defaults.conf
spark.yarn.archive hdfs://xiwu-cluster/spark/jars/spark-assembly-1.6.0-cdh5.14.2-hadoop2.6.0-cdh5.14.2.jar
spark.eventLog.enabled  true
spark.eventLog.dir      hdfs://xiwu-cluster/spark/log/envent-log
spark.serializer        org.apache.spark.serializer.KryoSerializer
spark.driver.memory     1g

修改hive-site.xml

<property>
  <name>hive.execution.engine</name>
  <value>spark</value>
</property>
<property>
  <name>hive.enable.spark.execution.engine</name>
  <value>true</value>
</property>
<property>
  <name>spark.home</name>
  <value>/root/spark-1.6.0-bin-hadoop2-without-hive</value>
</property>
<property>
  <name>spark.yarn.jar</name>
  <value>hdfs://xiwu-cluster/spark/jars/spark-assembly-1.6.0-cdh5.14.2-hadoop2.6.0-cdh5.14.2.jar</value>
</property>
<property>
  <name>spark.master</name>
  <value>yarn-cluster</value>
</property>
<property>
  <name>spark.serializer</name>
  <value>org.apache.spark.serializer.KryoSerializer</value>
</property>