4 examples of 'spark session in pyspark' in Python

Every line of 'spark session in pyspark' code snippets is scanned for vulnerabilities by our powerful machine learning engine that combs millions of open source libraries, ensuring your Python code is secure.

All examples are scanned by Snyk Code

By copying the Snyk Code Snippets you agree to
116def spark_context():
117 """
118 Returns the spark context.
119 """
120 return CommonSparkContext.Instance().sc()
6def main():
7 '''Program entry point'''
9 #Intialize a spark context
10 with pyspark.SparkContext("local", "PySparkWordCount") as sc:
11 #Get a RDD containing lines from this script file
12 lines = sc.textFile(__file__)
13 #Split each line into words and assign a frequency of 1 to each word
14 words = lines.flatMap(lambda line: line.split(" ")).map(lambda word: (word, 1))
15 #count the frequency for words
16 counts = words.reduceByKey(operator.add)
17 #Sort the counts in descending order based on the word frequency
18 sorted_counts = counts.sortBy(lambda x: x[1], False)
19 #Get an iterator over the counts to print a word and its frequency
20 for word,count in sorted_counts.toLocalIterator():
21 print(u"{} --> {}".format(word, count))
91def sc(self):
92 return self._sc
13def get_spark_context():
14 if SparkContext._active_spark_context:
15 return SparkContext._active_spark_context
16 else:
17 raise RuntimeError("SparkContext must be initialized")

Related snippets