1. 程式人生 > >shell指令碼載入資料檔案到hive表中

shell指令碼載入資料檔案到hive表中

如果執行時間允許,還可以增加判斷hive表是否存在的。

#!/bin/ksh
#-------------------------------------------------------------------------------------
#---1、shell載入資料
#---2、用hive_sql_handler呼叫寫好的sql指令碼
#-------------------------------------------------------------------------------------

DATE_YYYYMMDD=`date +%Y%m%d`
conf_file='/home/xinxinghe/cfg/datafile.properties'
tab_schema='temp' tab_fo017_reg="${tab_schema}.base_fo017_reg" tab_fo015_reg="${tab_schema}.base_fo015_reg" tab_fceso_reg="${tab_schema}.base_fceso_reg" tab_fo015_daily="${tab_schema}.base_fo015_active_daily" tab_fo015_monthly="${tab_schema}.base_fo015_active_monthly" dir_logfile="/home/xinxinghe/log/$(date +F)
"
###load_Data_to_Hive datafile tablename load_Data_to_Hive(){ datafile=$1 tabname=$2 ###判斷資料檔案是否為有效檔案 if [[ -f $datafile ]] then sqlstat="load data local inpath '${datafile}' overwrite into table ${tabname} partition (pt = '${DATE_YYYYMMDD}');" hive -e "$sqlstat
"
1>> $log_file 2>> $log_file else echo "WARNING(Datafile ${datafile} not Exists)!" >> $log_file #exit fi } ###begin ###判斷日誌目錄是否存在 if [[ ! -d $dir_logfile ]] then mkdir -p $dir_logfile fi log_file="${dir_logfile}/job_f_user_reg_load_data_$(date +'%Y%m%d%H%M%S%5N')" ###判斷配置檔案是否存在 if [[ -f $conf_file ]] then continue else echo "Configuration File $conf_file Not Exists!" >> $log_file exit fi ###讀取配置檔案 while read line || [[ -n $line ]] do typeset -L v1=$line typeset -R v2=$v1 line=$v2 if [[ ${#line} -ne 0 && ${line:0:1} != "#" ]] then prefix=${line%%=*} suffix=${line##*=} case $prefix in 'BASE_FO017_REG' ) file_fo017_reg=$suffix ;; 'BASE_FO015_REG' ) file_fo015_reg=$suffix ;; 'BASE_FCESO_REG' ) file_fceso_reg=$suffix ;; 'BASE_FO015_ACTIVE_DAILY' ) file_fo015_daily=$suffix ;; 'BASE_FO015_ACTIVE_MONTHLY' ) file_fo015_monthly=$suffix ;; 'DIR_NAME' ) dir_name=$suffix ;; esac fi done <"$conf_file" ###絕對路徑的資料檔案變數 file_fo017_reg="${dir_name}/${file_fo017_reg}" file_fo015_reg="${dir_name}/${file_fo015_reg}" file_fceso_reg="${dir_name}/${file_fceso_reg}" file_fo015_daily="${dir_name}/${file_fo015_daily}" file_fo015_monthly="${dir_name}/${file_fo015_monthly}" ###載入資料 echo "$file_fo017_reg $tab_fo017_reg" >> $log_file echo "$file_fo015_reg $tab_fo015_reg" >> $log_file echo "$file_fceso_reg $tab_fceso_reg" >> $log_file echo "$file_fo015_daily $tab_fo015_daily" >> $log_file echo "$file_fo015_monthly $tab_fo015_monthly" >> $log_file ###載入資料到表中 load_Data_to_Hive $file_fo017_reg $tab_fo017_reg load_Data_to_Hive $file_fo015_reg $tab_fo015_reg load_Data_to_Hive $file_fceso_reg $tab_fceso_reg load_Data_to_Hive $file_fo015_daily $tab_fo015_daily load_Data_to_Hive $file_fo015_monthly $tab_fo015_monthly ###執行hive指令碼 #/data1/hive/wls81/bin/hive_sql_handler.ksh /data1/hive/wls81/sql/f_user_reg.sql hive /home/xinxinghe/sql/f_user_reg.sql -d DATABASE_RESULT='temp' DATABASE_FBD='fbd_orc' 1>>$log_file 2>>$log_file