TidyMarble_JointTask copy.R 13.5 KB
Newer Older
Simon Ciranka's avatar
Simon Ciranka committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
#end# This script loads the messy Marble OCU data and makes it kind of tidy so that i can work
#with it further. The goal is to take the sql data which saves every trial seperatly and make it wider.
# i try tidyverse

#Makes Tidy Data from the now even more messy Marbledata but my coding skills get a little better at least.
library('tidyverse')
#library('here')
# this will load the marble date
directory<-here()
setwd(directory)
names<-c("startTime","rt","stimulus1","stimulus2","key_press","old_order","riskyKey","new_order","changed_Order","red_marbles","blue_marbles",
         "OtherChoseRisk","ChooseRisk","valueGamble","probGamble","Social1Ind0","payoff","cumulatedPayoff","valueSure","trialID","PercentBlueEstimate",
         "HowSure","test_part","sex","age","subject","time_elapsed","ProximityCrit","subjectNumber")

keeps<-c("startTime","rt","key_press","riskyKey","red_marbles","blue_marbles",
         "OtherChoseRisk","ChooseRisk","valueGamble","probGamble","Social1Ind0","payoff","cumulatedPayoff","valueSure","trialID","PercentBlueEstimate",
         "HowSure","test_part","sex","age","subject","subjectNumber")

MarbleData<-read.csv("A_RawData/MarbleOCU_Webdev.csv")
colnames(MarbleData)<-names
MarbleDataTibble<-as_tibble(MarbleData)
MarbleDataTibble$startTime<-as.Date(MarbleDataTibble$startTime)
MarbleDataTibble$subject=as.numeric(as.character(MarbleDataTibble$subject))

MarbleDataTibble[MarbleDataTibble$test_part=="SoloChoiceMarble",]$test_part="SoloChoice"

MarbleDataTibble%>%mutate(
  subject=case_when(
    (is.na(subject))~as.numeric(as.character(subjectNumber)),
    TRUE~subject
  )
)->MarbleDataTibble

#this is actually really cool
MarbleDataTibble[keeps] %>%                                               # check what i want to keep
  dplyr::filter(test_part=="SoloChoice" | test_part=="Solotrial" | test_part!="NULL") %>%
Simon Ciranka's avatar
Simon Ciranka committed
37
  dplyr::filter(subject %in% c(1:200))%>%# select only the parts I use for Dataanalysis
Simon Ciranka's avatar
Simon Ciranka committed
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
  dplyr::filter(rt!="NULL")-> tbl1 
# check what i want to keep

# i start with 3 bc i look back three items
for(i in 3:length(tbl1$rt)){
if(tbl1$PercentBlueEstimate[i-1]!="NULL" & tbl1$test_part[i]=="SoloChoice"){
  tbl1$PercentBlueEstimate[i]=tbl1$PercentBlueEstimate[i-1]
  }else if (tbl1$PercentBlueEstimate[i-2]!="NULL" & tbl1$test_part[i]=="SoloChoice"){
    tbl1$PercentBlueEstimate[i]=tbl1$PercentBlueEstimate[i-2]
  }
}
tbl1 %>%
  mutate(HowSure=lag(HowSure)) %>%
  na.omit() %>%dplyr::filter(test_part=="SoloChoice")->TidyMarbleSolo


# the same happens here for Social Choices
MarbleDataTibble[keeps] %>%                                               # check what i want to keep
  dplyr::filter(test_part=="SocialChoice" | test_part=="SocialTrial" | test_part!="NULL") %>%
Simon Ciranka's avatar
Simon Ciranka committed
57
  dplyr::filter(as.numeric(subject) %in% c(1:200))%>%# select only the parts I use for Dataanalysis
Simon Ciranka's avatar
Simon Ciranka committed
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
  dplyr::filter(rt!="NULL")-> tbl2                                # check what i want to keep

Subs<-unique(tbl2$subject)
#okay. this here must happen because jspsych saves the data from different trials in seperate rows
#in the following i just impute the NULLS that are listed in the Choice Row for Certainty reports as well as Frequency estimates.
TidyMarbleSocial<-tibble()

for(i in 3:length(tbl2$rt)){
  if(tbl2$PercentBlueEstimate[i-1]!="NULL" & tbl1$test_part[i]=="SocialChoice"){
    tbl2$PercentBlueEstimate[i]=tbl2$PercentBlueEstimate[i-1]
  }else if (tbl2$PercentBlueEstimate[i-2]!="NULL" & tbl2$test_part[i]=="SocialChoice"){
    tbl2$PercentBlueEstimate[i]=tbl2$PercentBlueEstimate[i-2]
  }
}
tbl2 %>%
  mutate(HowSure=lag(HowSure)) %>%
  na.omit() %>%dplyr::filter(test_part=="SocialChoice")->TidyMarbleSocial

TidyMarble<-dplyr::bind_rows(TidyMarbleSolo,TidyMarbleSocial)
##### ok ok. but now we have to filter out the pilot and see how this looks like.
TidyMarble$startTime=as.Date(TidyMarble$startTime)

TidyMarble<-TidyMarble[TidyMarble$startTime >= "2018-06-06",]


# i need three new coulms: number of marbles shown
# percent blue marbles shown, percent red marbles shown.
#first, i add an vector of nulls
#i define the additional columns before and add them afterwards.

TidyMarble$PercentBlueShownAbs<-NaN
TidyMarble$PercentRedShownAbs<-NaN
TidyMarble$PercentBlueShownRel<-NaN
TidyMarble$PercentRedShownRel<-NaN
TidyMarble$TotalNShown<-NaN
TidyMarble$Agegroup<-NaN

for(i in 1:length(TidyMarble$rt)){
  
  str_split(TidyMarble$blue_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/45->PercentBlueShownAbs
  
  str_split(TidyMarble$red_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/45->PercentRedShownAbs
  
  TidyMarble$PercentBlueShownAbs[i]<-as.numeric(PercentBlueShownAbs)
  TidyMarble$PercentRedShownAbs[i]<-as.numeric(PercentRedShownAbs)
  TidyMarble$TotalNShown[i]<-(PercentRedShownAbs*45)+(PercentBlueShownAbs*45)
  
  str_split(TidyMarble$blue_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/TidyMarble$TotalNShown[i]->PercentBlueShownRel
  
  str_split(TidyMarble$red_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/TidyMarble$TotalNShown[i]->PercentRedShownRel
  
  TidyMarble$PercentBlueShownRel[i]<-as.numeric(PercentBlueShownRel)
  TidyMarble$PercentRedShownRel[i]<-as.numeric(PercentRedShownRel)
  
  if(as.numeric(as.character(TidyMarble$age[i]))<18){
    TidyMarble$Agegroup[i]=1#are they adolescents?
Simon Ciranka's avatar
Simon Ciranka committed
126
    if(as.numeric(as.character(TidyMarble$age[i]))<=12){
Simon Ciranka's avatar
Simon Ciranka committed
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
      TidyMarble$Agegroup[i]=0#are they kids?
    }#enddoubleif.
  }else TidyMarble$Agegroup[i]=2#ore adults?
}

#finally; make index for Decicions from Description and expierience
TidyMarble$DFE1DFD0<-1
TidyMarble[TidyMarble$red_marbles==99,]$DFE1DFD0<-0
Webdev<-TidyMarble








######
######
###### HERE I LOAD THE VLAB DATA
######
######

names<-c("startTime","rt","stimulus1","stimulus2","key_press","old_order","riskyKey","new_order","changed_Order","red_marbles","blue_marbles",
         "OtherChoseRisk","ChooseRisk","valueGamble","probGamble","Social1Ind0","payoff","cumulatedPayoff","valueSure","trialID","PercentBlueEstimate",
         "HowSure","test_part","sex","age","subject","time_elapsed","ProximityCrit","subjectNumber")

keeps<-c("startTime","rt","key_press","riskyKey","red_marbles","blue_marbles",
         "OtherChoseRisk","ChooseRisk","valueGamble","probGamble","Social1Ind0","payoff","cumulatedPayoff","valueSure","trialID","PercentBlueEstimate",
         "HowSure","test_part","sex","age","subject","subjectNumber")

Simon Ciranka's avatar
Simon Ciranka committed
158
MarbleData<-read.csv("A_RawData/MarbleOCU_vlabNew.csv")
Simon Ciranka's avatar
Simon Ciranka committed
159 160 161 162 163 164 165 166 167 168 169 170
colnames(MarbleData)<-names
MarbleDataTibble<-as_tibble(MarbleData)

MarbleDataTibble$subject=as.numeric(as.character(MarbleDataTibble$subject))
MarbleDataTibble%>%mutate(
  subject=case_when(
    (is.na(subject))~as.numeric(as.character(subjectNumber)),
    TRUE~subject
  )
)->MarbleDataTibble

starttimes=unique(MarbleDataTibble[MarbleDataTibble$subject==153,]$startTime)
Simon Ciranka's avatar
Simon Ciranka committed
171 172 173
starttimes2=unique(MarbleDataTibble[MarbleDataTibble$subject==169,]$startTime)

#starttimes3=unique(MarbleDataTibble[MarbleDataTibble$subject==159,]$startTime)
Simon Ciranka's avatar
Simon Ciranka committed
174 175 176 177 178 179 180 181 182 183





MarbleDataTibble$test_part=as.character(MarbleDataTibble$test_part)

MarbleDataTibble[MarbleDataTibble$test_part=="SoloChoiceMarble",]$test_part="SoloChoice"

Sub153<-MarbleDataTibble[MarbleDataTibble$startTime==starttimes[length(starttimes)],]
Simon Ciranka's avatar
Simon Ciranka committed
184 185
Sub169<-MarbleDataTibble[MarbleDataTibble$startTime==starttimes2[length(starttimes2)],]

Simon Ciranka's avatar
Simon Ciranka committed
186 187
MarbleDataTibble=MarbleDataTibble[MarbleDataTibble$subject!=153,]
MarbleDataTibble=rbind(MarbleDataTibble,Sub153)
Simon Ciranka's avatar
Simon Ciranka committed
188 189 190
MarbleDataTibble=MarbleDataTibble[MarbleDataTibble$subject!=169,]
MarbleDataTibble=rbind(MarbleDataTibble,Sub169)

Simon Ciranka's avatar
Simon Ciranka committed
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
MarbleDataTibble$startTime=as.Date(MarbleDataTibble$startTime)
#this is actually really cool
MarbleDataTibble[keeps] %>%                                               # check what i want to keep
  dplyr::filter(test_part=="SoloChoice" | test_part=="Solotrial" | test_part!="NULL") %>%
  dplyr::filter(subject %in% c(1:230))%>%# select only the parts I use for Dataanalysis
  dplyr::filter(rt!="NULL")-> tbl1 
# check what i want to keep
Subs<-unique(tbl1$subject)




for(i in 3:length(tbl1$rt)){
  if(tbl1$PercentBlueEstimate[i-1]!="NULL" & tbl1$test_part[i]=="SoloChoice"){
    tbl1$PercentBlueEstimate[i]=tbl1$PercentBlueEstimate[i-1]
  }else if (tbl1$PercentBlueEstimate[i-2]!="NULL" & tbl1$test_part[i]=="SoloChoice"){
    tbl1$PercentBlueEstimate[i]=tbl1$PercentBlueEstimate[i-2]
  }
}
tbl1 %>%
  mutate(HowSure=lag(HowSure)) %>%
  na.omit() %>%dplyr::filter(test_part=="SoloChoice")->TidyMarbleSolo


# the same happens here for Social Choices
MarbleDataTibble[keeps] %>%                                               # check what i want to keep
  dplyr::filter(test_part=="SocialChoice" | test_part=="SocialTrial" | test_part!="NULL") %>%
  dplyr::filter(as.numeric(subject) %in% c(1:230))%>%# select only the parts I use for Dataanalysis
  
  
  dplyr::filter(rt!="NULL")-> tbl2                                # check what i want to keep

Subs<-unique(tbl2$subject)
#okay. this here must happen because jspsych saves the data from different trials in seperate rows
#in the following i just impute the NULLS that are listed in the Choice Row for Certainty reports as well as Frequency estimates.
TidyMarbleSocial<-tibble()


for(i in 3:length(tbl2$rt)){
  if(tbl2$PercentBlueEstimate[i-1]!="NULL" & tbl1$test_part[i]=="SocialChoice"){
    tbl2$PercentBlueEstimate[i]=tbl2$PercentBlueEstimate[i-1]
  }else if (tbl2$PercentBlueEstimate[i-2]!="NULL" & tbl2$test_part[i]=="SocialChoice"){
    tbl2$PercentBlueEstimate[i]=tbl2$PercentBlueEstimate[i-2]
  }
}
tbl2 %>%
  mutate(HowSure=lag(HowSure)) %>%
  na.omit() %>%dplyr::filter(test_part=="SocialChoice")->TidyMarbleSocial

TidyMarble<-dplyr::bind_rows(TidyMarbleSolo,TidyMarbleSocial)

# only keep these ones that started after the pilot
DidIFindIt<-TidyMarble[TidyMarble$startTime >= "2018-06-06",]

# check if everything is correct
hist(DidIFindIt$subject,breaks=length(unique(DidIFindIt$subject)))
Subz<-unique(DidIFindIt$subject)
for (i in 1:length(unique(DidIFindIt$subject))){
  print(length(DidIFindIt[DidIFindIt$subject==Subz[i],]$subject))
}

TidyMarble<-TidyMarble[TidyMarble$startTime >= "2018-06-06",]


# i need three new coulms: number of marbles shown
# percent blue marbles shown, percent red marbles shown.
#first, i add an vector of nulls
#i define the additional columns before and add them afterwards.

TidyMarble$PercentBlueShownAbs<-NaN
TidyMarble$PercentRedShownAbs<-NaN
TidyMarble$PercentBlueShownRel<-NaN
TidyMarble$PercentRedShownRel<-NaN
TidyMarble$TotalNShown<-NaN
TidyMarble$Agegroup<-NaN

for(i in 1:length(TidyMarble$rt)){
  
  str_split(TidyMarble$blue_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/45->PercentBlueShownAbs
  
  str_split(TidyMarble$red_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/45->PercentRedShownAbs
  
  TidyMarble$PercentBlueShownAbs[i]<-as.numeric(PercentBlueShownAbs)
  TidyMarble$PercentRedShownAbs[i]<-as.numeric(PercentRedShownAbs)
  TidyMarble$TotalNShown[i]<-(PercentRedShownAbs*45)+(PercentBlueShownAbs*45)
  
  str_split(TidyMarble$blue_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/TidyMarble$TotalNShown[i]->PercentBlueShownRel
  
  str_split(TidyMarble$red_marbles[i],pattern=",")%>%
    unlist()%>%
    as.numeric()%>%
    sum()/TidyMarble$TotalNShown[i]->PercentRedShownRel
  
  TidyMarble$PercentBlueShownRel[i]<-as.numeric(PercentBlueShownRel)
  TidyMarble$PercentRedShownRel[i]<-as.numeric(PercentRedShownRel)
  
  if(as.numeric(as.character(TidyMarble$age[i]))<18){
    TidyMarble$Agegroup[i]=1#are they adolescents?
Simon Ciranka's avatar
Simon Ciranka committed
298
    if(as.numeric(as.character(TidyMarble$age[i]))<=12){
Simon Ciranka's avatar
Simon Ciranka committed
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
      TidyMarble$Agegroup[i]=0#are they kids?
    }#enddoubleif.
  }else TidyMarble$Agegroup[i]=2#ore adults?
}

#finally; make index for Decicions from Description and expierience
TidyMarble$DFE1DFD0<-1
TidyMarble[TidyMarble$red_marbles==99,]$DFE1DFD0<-0

vLab<-TidyMarble


#Wow that was a fucking mess!
TidyMarble<-rbind(Webdev,vLab)


Supposed<-c(1:230)

subs<-sort(as.numeric(as.character(unique(TidyMarble$subject))))
missing<-(Supposed %in% subs)
missingSubIdx<-Supposed[!missing]

TidyMarble$age<-as.numeric(as.character(TidyMarble$age))

TidyMarble<-TidyMarble%>%filter(age>8)%>%select( colnames(TidyMarble)[colnames(TidyMarble)!="subjectNumber" & colnames(TidyMarble)!="trialID"] )



trials<-rep(1,length(subs))

for(i in 1:length(subs)){
  trials[i]<-length(TidyMarble[TidyMarble$subject==subs[i],]$subject)
  print(trials[i])
}

hist(trials) 

TooManyTrials<-subs[trials>144]# ok these are all kids. Something must have went really wrong here. Anyway. I only keep the ones, older than 

for (i in 1:length(TooManyTrials)){
  print(unique(TidyMarble[TidyMarble$subject==TooManyTrials[i],]$age))
}
# they colleceted sub 153 twice. i only keep the last 

temp<-TidyMarble[TidyMarble$subject %in% TooManyTrials & TidyMarble$age<15,]
TidyMarble<-TidyMarble%>%filter(!(subject %in% TooManyTrials))%>%rbind(temp)%>%arrange(subject)
notEnoughTrials<-subs[trials<143]# ok these are all kids. Something must have went really wrong here. Anyway. I only keep the ones, older than 
TidyMarble<-TidyMarble%>%filter(!(subject %in% notEnoughTrials))

NewSubs=unique(TidyMarble$subject)
NewTrials=NewSubs

for(i in 1:length(NewSubs)){
  NewTrials[i]<-length(TidyMarble[TidyMarble$subject==NewSubs[i],]$subject)
}

hist(TidyMarble$subject,breaks=length(unique(TidyMarble$subject))*3)

recollect=(Supposed %in% unique(TidyMarble$subject))
missing<-Supposed[!recollect]

write.csv(TidyMarble,file="TidyMarbleNew.csv")
#down here i build sth for Lucas so we can connect datasets.
demographics<-c("sex","age","subject")
demographicsTBL<-TidyMarble[demographics]
Lucas<-unique(demographicsTBL)
save(Lucas,file="Demographics.RData")