spark python例子
阿新 • • 發佈:2019-01-07
# -*- coding: utf-8 -*- from __future__ import print_function import sys from operator import add from pyspark import SparkContext #./pyspark /home/yunshouhu/PycharmProjects/untitled/word.py /home/yunshouhu/PycharmProjects/untitled/data.txt if __name__=="__main__": if len(sys.argv)!=2: print ("useg: word <file>",file=sys.stderr); exit(-1) sc=SparkContext(appName="myword") lines=sc.textFile(sys.argv[1],1) counts=lines.flatMap(lambda x:x.split(' ')).map(lambda x:(x,1)).reduceByKey(add) output=counts.collect(); for (word,count) in output: print("來自中國: %s %i" % (word,count)) sc.stop()