forked from ScholaNest/Spark-Programming-In-Python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathUtils.py
More file actions
20 lines (18 loc) · 721 Bytes
/
Copy pathUtils.py
File metadata and controls
20 lines (18 loc) · 721 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from pyspark.sql import SparkSession
from lib.ConfigLoader import get_spark_conf
def get_spark_session(env):
if env == "LOCAL":
return SparkSession.builder \
.config(conf=get_spark_conf(env)) \
.config('spark.sql.autoBroadcastJoinThreshold',-1) \
.config('spark.sql.adaptive.enabled','false') \
.config('spark.driver.extraJavaOptions',
'-Dlog4j.configuration=file:log4j.properties') \
.master("local[2]") \
.enableHiveSupport() \
.getOrCreate()
else:
return SparkSession.builder \
.config(conf=get_spark_conf(env)) \
.enableHiveSupport() \
.getOrCreate()