forked from ScholaNest/Spark-Programming-In-Python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathHelloSpark.py
More file actions
29 lines (22 loc) · 699 Bytes
/
Copy pathHelloSpark.py
File metadata and controls
29 lines (22 loc) · 699 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import sys
from pyspark.sql import *
from lib.logger import Log4j
from lib.utils import *
if __name__ == "__main__":
conf = get_spark_app_config()
spark = SparkSession \
.builder \
.appName("HelloSpark") \
.master("local[2]") \
.getOrCreate()
logger = Log4j(spark)
if len(sys.argv) != 2:
logger.error("Usage: HelloSpark <filename>")
sys.exit(-1)
logger.info("Starting HelloSpark")
survey_raw_df = load_survey_df(spark, sys.argv[1])
partitioned_survey_df = survey_raw_df.repartition(2)
count_df = count_by_country(partitioned_survey_df)
count_df.show()
logger.info("Finished HelloSpark")
spark.stop()