assignment2

+(u'allston', 207)
+(u'area', 6)
+(u'back', 283)
+(u'bay', 302)
+(u'beacon', 205)
+(u'boston', 348)
+(u'brighton', 180)
+(u'charlestown', 65)
+(u'chinatown', 55)
+(u'district', 8)
+(u'dorchester', 240)
+(u'downtown', 152)
+(u'east', 126)
+(u'end', 518)
+(u'fenway', 296)
+(u'hill', 306)
+(u'hyde', 26)

+(u'jamaica', 315)
+(u'leather', 8)
+(u'longwood', 6)
+(u'mattapan', 20)
+(u'medical', 6)
+(u'mission', 101)
+(u'neighborhood', 1)
+(u'north', 132)
+(u'park', 26)
+(u'plain', 315)
+(u'roslindale', 56)
+(u'roxbury', 177)
+(u'south', 550)
+(u'village', 19)
+(u'waterfront', 69)
+(u'west', 88)

+import sys
+import re
+from operator import add
+
+from pyspark import SparkContext
+
+def map_phase(x):
+	x = re.sub('--', ' ', x)
+	x = re.sub("'", '', x)
+	return re.sub('[?!@#$\'",.;:()]', '', x).lower()
+
+def countWord(line):
+	global count_number
+	if (line == "Tokyo"):
+		count_number += 1
+	return line.split(' ')
+
+if __name__ == "__main__":
+	if len(sys.argv) < 4:
+		print >> sys.stderr, "Usage: wordcount <master> <inputfile> <outputfile>"
+       		exit(-1)
+	sc = SparkContext(sys.argv[1], "python_wordcount_sorted in bigdataprogrammiing")
+	lines = sc.textFile(sys.argv[2],2)
+	count_number = sc.accumulator(0)
+	
+	counts = lines\
+	.flatMap(countWord)\
+	.filter(lambda x: x!="Tokyo")\
+	.map(lambda x: (x.lower(), 1))\
+	.reduceByKey(lambda x,y:x+y)\
+	.sortByKey(ascending=True)	
+#.sortBy(lambda x: x[0])
+    
+	counts.saveAsTextFile("hdfs://localhost:9000/output5")
+	print('Number of Tokyo : ', count_number.value)
+  	sc.stop()