1. 程式人生 > >叢集間傳輸資料

叢集間傳輸資料

#!/bin/bash

# data train 1.0
# CREATED BY ZJF @2018-08-01 

# source env
source /etc/profile
export JAVA_HOME=/home/work/soft/jdk1.7.0_55

# args check
if [ $# -eq 2 ]
then
    echo "the args is right, caculating..."
else
    echo -e "\n\nERROR! \nthe args is wrong ,you should give 2 params, one aim hdfs path and anothor destination hdfs path, \nlike '/user/h_data_platform/platform/gamesdk/dwm_gc_v2_comic_action/date=2018-08-01/ /user/h_data_platform/platform/gamesdk/dw
m_gc_v2_comic_action/date=2018-08-01/'\n\n"
exit 1; fi # set workdir WORK_DIR=$(cd `dirname $0`; pwd) echo ${WORK_DIR} cd ${WORK_DIR} # transfor data between clusters # use infra-client, must. /home/work/soft/infra-client/bin/hadoop --cluster zjyprc-hadoop fs -mkdir -p $2; /home/work/soft/infra-client/bin/hadoop --cluster zjyprc-hadoop distcp -D mapreduce.job.queuename=root.production.game_group.gamepublish.game_publish_queue -prpc -m 50
-ba ndwidth 1 -ignoreDeleted -overwrite hdfs://c3prc-hadoop${1} hdfs://zjyprc-hadoop${2}