1. 程式人生 > >Spark中的各種action運算元操作(scala版)

Spark中的各種action運算元操作(scala版)

這裡直接貼程式碼了,action的介紹都在java那裡。
package cn.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

object ActionOperation {
def main(args: Array[String]): Unit = {
//reduceTest()
// collectTest()
//takeTest()
//countTest()
countByKeyTest()
}

def reduceTest(){
val conf = new SparkConf()
.setAppName(“reduce”)
.setMaster(“local”)
val sc = new SparkContext(conf)

val list = Array(1,2,3,4,5,6,7,8,9,10)

val numbersRDD = sc.parallelize(list, 1)
val sum = numbersRDD.reduce(_+_)
println(sum)

}

def collectTest(){
val conf = new SparkConf()
.setAppName(“collect”)
.setMaster(“local”)
val sc = new SparkContext(conf)

val list = Array(1,2,3,4,5,6,7,8,9,10)

val numbersRDD = sc.parallelize(list, 1)
val doubleNumber = numbersRDD.map { num => num*2 }
val doubleList = doubleNumber.collect()
for(num <- doubleList){
  println(num)
}

}

def takeTest(){
val conf = new SparkConf()
.setAppName(“take”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val list = Array(1,2,3,4,5,6,7,8,9,10)

val numbersRDD = sc.parallelize(list, 1)

val top3 = numbersRDD.take(3)
for(num <- top3){
  println(num)
}

}

def countTest(){
val conf = new SparkConf()
.setAppName(“count”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val list = Array(1,2,3,4,5,6,7,8,9,10)

val numbersRDD = sc.parallelize(list, 1)
val count = numbersRDD.count()
println(count)

}

def saveAsTextFileTest(){
val conf = new SparkConf()
.setAppName(“saveAsTextFile”)
.setMaster(“local”)
val sc = new SparkContext(conf)
val linesRDD = sc.textFile(“”, 1)

linesRDD.saveAsTextFile("hdfs://spark1:9000/spark.txt")

}

def countByKeyTest(){
val conf = new SparkConf()
.setAppName(“countByKey”)
.setMaster(“local”)
val sc = new SparkContext(conf)

val studentList = Array(("class1","leo"),
                        ("class2","jack"),
                        ("class1","marry"),
                        ("class2","ksc"),
                        ("class2","my"))
val studentsRDD = sc.parallelize(studentList, 1)
val studentsCount = studentsRDD.countByKey()
for((k,v) <- studentsCount){
  println(k+":"+v)
}

}
}

這裡不得不說scala的函數語言程式設計與各種特性使得scala的程式比java簡潔的多,但是對於我這裡菜鳥來說scala的特性學習起來真的很麻煩。