RUGS-Meetup/2a.cleanData_model.R at master · thiakx/RUGS-Meetup · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
#I will setwd("/MY_Working_Directory/data") where the kaggle train/test data are placed
source("../baseFunctions_cleanData.R")

trainData<-read.csv("train_wHex.csv")
testData<-read.csv("test_wHex.csv")

trainData2<-trainData[,!(names(trainData) %in% c("latitude","longitude"))]

#replace string NA with unknown
trainData2$tag_type<-as.character(trainData2$tag_type)
trainData2$tag_type[is.na(trainData2$tag_type)]<-"unknown"
trainData2$tag_type<-as.factor(trainData2$tag_type)

testData$tag_type<-as.character(testData$tag_type)
testData$tag_type[is.na(testData$tag_type)]<-"unknown"
testData$tag_type<-as.factor(testData$tag_type)

trainData2$source<-as.character(trainData2$source)
trainData2$source[is.na(trainData2$source)]<-"unknown"
trainData2$source<-as.factor(trainData2$source)

testData$source<-as.character(testData$source)
testData$source[is.na(testData$source)]<-"unknown"
testData$source<-as.factor(testData$source)

#get data table of median values by city, tag type
trainData2$num_views<-as.numeric(trainData2$num_views)
trainData2$num_votes<-as.numeric(trainData2$num_votes)
trainData2$num_comments<-as.numeric(trainData2$num_comments)

trainData2$month<-month(trainData2$created_time)
trainData2$year<-year(trainData2$created_time)

#remove first 10 month 2012 data as they have higher d from 2013 Apr
#remove march 2013 as it has high d as well.
#d is like the distance of difference, smaller d = the two data sets are closer
# ksTest<-ks.test(trainData2$num_views[trainData2$month==4&trainData2$year==2013],
#                 trainData2$num_views[trainData2$month==9&trainData2$year==2012])
trainData2<-trainData2[trainData2$month==11|trainData2$month==12|trainData2$year==2013,]
trainData2<-trainData2[trainData2$month!=3,]

#no longer need the date cols
#test data drop year and month later down the code
trainData2$created_time<-NULL
# trainData2$year<-NULL
# trainData2$month<-NULL

#combine the remote_API tags with tag_type
trainData2<-remapAPI(trainData2)
testData2<-testData[,!(names(testData) %in% c("latitude","longitude","created_time"))]
testData2<-remapAPI(testData2)

# #chicargo remote_api_created is one of the kind weird and should be modelled seperately
# ksTest_city<- ks.test(trainData2$num_views[trainData2$city=="oakland"],
#               trainData2$num_views[trainData2$city=="chicargo"&trainData2$source!="remote_api_created"])
#
# ksTest_city2<-ks.test(trainData2$num_views[trainData2$city=="chicargo"],
#               trainData2$num_views[trainData2$city=="chicargo"&trainData2$source!="remote_api_created"])
#
# #richmond, new_haven have d=0 which is strange
# ksTest_city3<-ks.test(trainData2$num_views[trainData2$city=="new_haven"],
#         trainData2$num_views[trainData2$city=="new_haven"&trainData2$source!="remote_api_created"])

trainDataMod<-trainData2
trainDataMod<-trainDataMod[!(trainDataMod$city=="chicargo"&trainDataMod$source=="remote_api_created"),]
testDataMod<-testData2[!(testData2$city=="chicargo"&testData2$source=="remote_api_created"),]
trainDataMod<-data.table(trainDataMod)

#remove summary/desc
trainData2<-trainData2[,!(names(trainData2) %in% c("summary","description"))]
testData2<-testData2[,!(names(testData2) %in% c("summary","description"))]

#remove train data more than 3 Median Absolute Deviation away from median (outliers)
trainDataMod<-madRemove(trainDataMod,3)

#convert traindataRf back to data frame
trainDataMod<-data.frame(trainDataMod)

#log view (it has large variation)
trainDataMod$num_views<-log(trainDataMod$num_views+1)

#relevel test data to match train data tags, source. All NAs will be estimated using median in modeling
testDataMod$tag_type<-factor(testDataMod$tag_type,levels=levels(trainDataMod$tag_type))
testDataMod$source<-factor(testDataMod$source,levels=levels(trainDataMod$source))
testDataMod<-testDataMod[!is.na(testDataMod$tag_type),]
testDataMod<-testDataMod[!is.na(testDataMod$source),]

#reclassify text + split tags
trainDataMod<-wordMine(trainDataMod,"all")
testCol<-names(trainDataMod)
drops <- c("num_votes","num_comments","num_views","year","month")
testCol<-testCol[!testCol %in% drops]
testDataMod<-wordMine(testDataMod,testCol)
trainDataMod<-trainDataMod[,!(names(trainDataMod) %in% c("summary","description"))]
testDataMod<-testDataMod[,!(names(testDataMod) %in% c("summary","description"))]

#save the data
save(trainDataMod,file="trainDataMod.Rdata")
save(testDataMod,file="testDataMod.Rdata")
save(trainData2,file="trainData2.Rdata")
save(testData2,file="testData2.Rdata")