1. 程式人生 > >Spark 配置及提交應用

Spark 配置及提交應用

export SPARK_MASTER_WEBUI_PORT=8888
export SPARK_CLASSPATH=$SPARK_CLASSPATH:/home/xxx/sparkapp/lib/mysql-connector-java-5.1.27.jar:/home/xxx/sparkapp/lib/hbase-common-1.2.0.jar:/home/xxx/sparkapp/lib/hbase-client-1.2.0.jar
export JAVA_HOME=/home/xxxx/jdk1.8.0_181
export HADOOP_HOME=/home//hadoop-2.6.0-cdh5.8.0
export SCALA_HOME=/home/xxx/scala-2.12.1
export SPARK_MASTER_IP=Trade-Test-Spark001
export SPARK_MASTER_HOST=Trade-Test-Spark001
export SPARK_LOCAL_IP=Trade-Test-Spark001
export SPARK_WORKER_MEMORY=950m
export SPARK_HOME=/home/xxx/spark-2.2.0-bin-hadoop2.6
export SPARK_WORKER_CORES=2
export SPARK_WORKER_INSTANCES=1
export HADOOP_CONF_DIR=/home/xxx/hadoop-2.6.0-cdh5.8.0/etc/hadoop

 

 

#

/home/xxxx/spark-2.2.0-bin-hadoop2.6/bin/spark-submit --master spark://Trade-Test-Spark001:6066  --deploy-mode cluster --class com.xxx.RecommendationApplication   --driver-memory 700m --executor-memory 500m --num-executors 1   /home/xxx/sparkJar/recommendation.jar