shell指令碼載入資料檔案到hive表中
阿新 • • 發佈:2018-12-22
如果執行時間允許,還可以增加判斷hive表是否存在的。
#!/bin/ksh
#-------------------------------------------------------------------------------------
#---1、shell載入資料
#---2、用hive_sql_handler呼叫寫好的sql指令碼
#-------------------------------------------------------------------------------------
DATE_YYYYMMDD=`date +%Y%m%d`
conf_file='/home/xinxinghe/cfg/datafile.properties'
tab_schema='temp'
tab_fo017_reg="${tab_schema}.base_fo017_reg"
tab_fo015_reg="${tab_schema}.base_fo015_reg"
tab_fceso_reg="${tab_schema}.base_fceso_reg"
tab_fo015_daily="${tab_schema}.base_fo015_active_daily"
tab_fo015_monthly="${tab_schema}.base_fo015_active_monthly"
dir_logfile="/home/xinxinghe/log/$(date +F) "
###load_Data_to_Hive datafile tablename
load_Data_to_Hive(){
datafile=$1
tabname=$2
###判斷資料檔案是否為有效檔案
if [[ -f $datafile ]]
then
sqlstat="load data local inpath '${datafile}' overwrite into table ${tabname} partition (pt = '${DATE_YYYYMMDD}');"
hive -e "$sqlstat " 1>> $log_file 2>> $log_file
else
echo "WARNING(Datafile ${datafile} not Exists)!" >> $log_file
#exit
fi
}
###begin
###判斷日誌目錄是否存在
if [[ ! -d $dir_logfile ]]
then
mkdir -p $dir_logfile
fi
log_file="${dir_logfile}/job_f_user_reg_load_data_$(date +'%Y%m%d%H%M%S%5N')"
###判斷配置檔案是否存在
if [[ -f $conf_file ]]
then
continue
else
echo "Configuration File $conf_file Not Exists!" >> $log_file
exit
fi
###讀取配置檔案
while read line || [[ -n $line ]]
do
typeset -L v1=$line
typeset -R v2=$v1
line=$v2
if [[ ${#line} -ne 0 && ${line:0:1} != "#" ]]
then
prefix=${line%%=*}
suffix=${line##*=}
case $prefix in
'BASE_FO017_REG' )
file_fo017_reg=$suffix
;;
'BASE_FO015_REG' )
file_fo015_reg=$suffix
;;
'BASE_FCESO_REG' )
file_fceso_reg=$suffix
;;
'BASE_FO015_ACTIVE_DAILY' )
file_fo015_daily=$suffix
;;
'BASE_FO015_ACTIVE_MONTHLY' )
file_fo015_monthly=$suffix
;;
'DIR_NAME' )
dir_name=$suffix
;;
esac
fi
done <"$conf_file"
###絕對路徑的資料檔案變數
file_fo017_reg="${dir_name}/${file_fo017_reg}"
file_fo015_reg="${dir_name}/${file_fo015_reg}"
file_fceso_reg="${dir_name}/${file_fceso_reg}"
file_fo015_daily="${dir_name}/${file_fo015_daily}"
file_fo015_monthly="${dir_name}/${file_fo015_monthly}"
###載入資料
echo "$file_fo017_reg $tab_fo017_reg" >> $log_file
echo "$file_fo015_reg $tab_fo015_reg" >> $log_file
echo "$file_fceso_reg $tab_fceso_reg" >> $log_file
echo "$file_fo015_daily $tab_fo015_daily" >> $log_file
echo "$file_fo015_monthly $tab_fo015_monthly" >> $log_file
###載入資料到表中
load_Data_to_Hive $file_fo017_reg $tab_fo017_reg
load_Data_to_Hive $file_fo015_reg $tab_fo015_reg
load_Data_to_Hive $file_fceso_reg $tab_fceso_reg
load_Data_to_Hive $file_fo015_daily $tab_fo015_daily
load_Data_to_Hive $file_fo015_monthly $tab_fo015_monthly
###執行hive指令碼
#/data1/hive/wls81/bin/hive_sql_handler.ksh /data1/hive/wls81/sql/f_user_reg.sql
hive /home/xinxinghe/sql/f_user_reg.sql -d DATABASE_RESULT='temp' DATABASE_FBD='fbd_orc' 1>>$log_file 2>>$log_file