Initial Commit

Author: LearningJournal

UDFDemo/UDFDemo.py

@@ -31,14 +31,18 @@ def parse_gender(gender):
         .option("inferSchema", "true") \
         .csv("data/survey.csv")
 
-    survey_df.show()
+    survey_df.show(10)
 
     parse_gender_udf = udf(parse_gender, returnType=StringType())
+    logger.info("Catalog Entry:")
     [logger.info(r) for r in spark.catalog.listFunctions() if "parse_gender" in r.name]
+
     survey_df2 = survey_df.withColumn("Gender", parse_gender_udf("Gender"))
-    survey_df2.show()
+    survey_df2.show(10)
 
     spark.udf.register("parse_gender_udf", parse_gender, StringType())
+    logger.info("Catalog Entry:")
     [logger.info(r) for r in spark.catalog.listFunctions() if "parse_gender" in r.name]
+
     survey_df3 = survey_df.withColumn("Gender", expr("parse_gender_udf(Gender)"))
-    survey_df3.show()
+    survey_df3.show(10)