forked from ScholaNest/Spark-Programming-In-Python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathMiscDemo.py
More file actions
38 lines (31 loc) · 1.44 KB
/
Copy pathMiscDemo.py
File metadata and controls
38 lines (31 loc) · 1.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, monotonically_increasing_id, when, expr
from pyspark.sql.types import *
from lib.logger import Log4j
if __name__ == "__main__":
spark = SparkSession \
.builder \
.appName("Misc Demo") \
.master("local[2]") \
.getOrCreate()
logger = Log4j(spark)
data_list = [("Ravi", "28", "1", "2002"),
("Abdul", "23", "5", "81"), # 1981
("John", "12", "12", "6"), # 2006
("Rosy", "7", "8", "63"), # 1963
("Abdul", "23", "5", "81")] # 1981
raw_df = spark.createDataFrame(data_list).toDF("name", "day", "month", "year").repartition(3)
raw_df.printSchema()
final_df = raw_df.withColumn("id", monotonically_increasing_id()) \
.withColumn("day", col("day").cast(IntegerType())) \
.withColumn("month", col("month").cast(IntegerType())) \
.withColumn("year", col("year").cast(IntegerType())) \
.withColumn("year", when(col("year") < 20, col("year") + 2000)
.when(col("year") < 100, col("year") + 1900)
.otherwise(col("year"))) \
.withColumn("dob", expr("to_date(concat(day,'/',month,'/',year), 'd/M/y')")) \
.drop("day", "month", "year") \
.dropDuplicates(["name", "dob"]) \
# .sort(expr("dob desc")) This doesn't seem to be working
.sort(col("dob").desc())
final_df.show()