4 examples of 'spark session in pyspark' in Python

116
def spark_context():
117
    """
118
    Returns the spark context.
119
    """
120
    return CommonSparkContext.Instance().sc()

6
def main():
7
    '''Program entry point'''
8
9
    #Intialize a spark context
10
    with pyspark.SparkContext("local", "PySparkWordCount") as sc:
11
        #Get a RDD containing lines from this script file  
12
        lines = sc.textFile(__file__)
13
        #Split each line into words and assign a frequency of 1 to each word
14
        words = lines.flatMap(lambda line: line.split(" ")).map(lambda word: (word, 1))
15
        #count the frequency for words
16
        counts = words.reduceByKey(operator.add)
17
        #Sort the counts in descending order based on the word frequency
18
        sorted_counts =  counts.sortBy(lambda x: x[1], False)
19
        #Get an iterator over the counts to print a word and its frequency
20
        for word,count in sorted_counts.toLocalIterator():
21
            print(u"{} --> {}".format(word, count))

91
def sc(self):
92
    return self._sc

13
def get_spark_context():
14
    if SparkContext._active_spark_context:
15
        return SparkContext._active_spark_context
16
    else:
17
        raise RuntimeError("SparkContext must be initialized")