FormattingFixCountForMachineLearning

#Load Packages
library("broom", lib.loc="~/R/R-3.3.1/library")
library("tidyr", lib.loc="~/R/R-3.3.1/library")
library("ggplot2", lib.loc="~/R/R-3.3.1/library")
library("gridExtra", lib.loc="~/R/R-3.3.1/library")
library("dplyr", lib.loc="~/R/R-3.3.1/library")
library("tidyr", lib.loc="~/R/R-3.3.2/library")
library("plotly", lib.loc="~/R/R-3.3.2/library")
library(plyr)
library(data.table)
library(reshape2)
library(readr)
library(jpeg)


#Goals of this code
#1. Import all fixation and saccade reports
#2. Combine them 
#3. Figure out how to add them to a folder based on column "SubjectID_imageID"


sdif = 0
y =37


#Some vestigal bits of code bc changed purpose halfway through when talking to Pradeep. 

#Loading Sample data####
#note, 9a and 9b were combined in advance to this. 9a block 11 crashed part way through. 
#part of this loop subsets to only include -1 in test confidence column. The purpose of this is some glitchy trials appear in the study period with only 1 sample from a test image which comes up as a new image but it's fake). 

mylistnew <- list()
for (i in 1:y){
  i = i + sdif #this is to adjust what subject you want to import
  data_dir = 'C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/fixationreports' 
  filename =  i
  fileextension = ".csv"
  filename = paste0(filename, fileextension)
  filepath = paste(data_dir, '/', filename, sep = ''); filepath
  
  tosubset <- fread(filepath)
  
  newdf <- tosubset
  
  #getting rid of "Trial: " so trial number can be numeric (mainly to match the df.merged_wdp)
  newdf$TRIAL_LABEL <- gsub("Trial: ", "", newdf$TRIAL_LABEL)
  newdf$TRIAL_LABEL <- as.numeric(newdf$TRIAL_LABEL)
  i = i -sdif #(part 2 to not have a list 9 items long for subj 7-9)
  mylistnew[[i]] <- newdf
 
}

#Load data 2####
#Loading in data from other script (Objfam_qualitychecks_accuracy_betweenSubj_april11) If it's still named that. 
df.merged_wdp <- fread('C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/dp_and_subqAcc.csv')

# #1.5 Fixing Studyphase/repnum#### 
# #selecting only study 
# df.merged_wdp <- ifelse(df.merged_wdp)
# 
# df.merged_wdp <- subset(df.merged_wdp, subset = df.merged_wdp$task == 'study')
# df.merged_wdp$repnum <- with(df.merged_wdp, ave(RECORDING_SESSION_LABEL, RECORDING_SESSION_LABEL, image_filename, FUN = seq_along))
# 
# #does repnum equal studyphase for the subjects that had it?
# df.merged_wdp$
# 
# df.merged_wdp$errorcheck <- ifelse(df.merged_wdp$studyphase==df.merged_wdp$repnum, "good", "maybeerror")
# 


#2. Label each row as hit/miss/FA/CR and dprime
#practice not in a loop
#making a vector of relevant columns I want from df.merged_wdp
df.merged_wdp$imgname <- substr(df.merged_wdp$image_filename, 1,7)
df.merged_wdp$TRIAL_LABEL <- gsub("Trial: ", "", df.merged_wdp$TRIAL_LABEL)
df.merged_wdp$TRIAL_LABEL <- as.numeric(df.merged_wdp$TRIAL_LABEL)
df.sorted <- df.merged_wdp[order(SID, imgname, TRIAL_LABEL),]
df.sorted <- subset(df.sorted, subset = df.sorted$task == 'study')
df.sorted$IID_SID <- paste0(df.sorted$imgname,"_._", df.sorted$SID)

RelCol <- c("IID_SID","SID", "RECORDING_SESSION_LABEL", "X__Reaction_Time__1", "block", "imgname", "studyphase",  "oldnearorfar", "test_confidence.x", "TRIAL_LABEL", "SubsequentAccuracy", "SubsequentMorph", "test_confidence.y", "dprime_nearonly_noguess", "dprime_faronly_noguess")

#Loop to merge d' and hit/miss (and many other rows)
mylist2 <- list(seq(1:y))
j = 1 
for (j in 1:y) {
  mylist2[[j]] <- merge(mylistnew[[j]], df.sorted[ , RelCol, with = F], by = c('RECORDING_SESSION_LABEL', 'TRIAL_LABEL'), all = F)
} 

#2.5. set it to show only the used eye's location
#in loop for all subj
mylist3 <- list(seq(1:y)) 
k = 1
for (k in 1:y) {
  #Setting list number to df named visualizermerged
  mylist2[[k]] ->  visualizerMerged
  #Doing nothing bc not relevant for fixation and saccade reports
 
  visualizerMerged -> mylist3[[k]] 
}


#3. Loop to save with everything in its own folder by trial...

#Encoding####
#saving individual trial to it's own folder.
for (m in 1:y){
  mylist3[[m]] -> alltrials
  for (n in 1:length(unique(alltrials$TRIAL_LABEL))) {
    trial1 <- subset(alltrials, subset = alltrials$TRIAL_LABEL == n)
    s1 <- trial1[1, 'IID_SID']
    i1 <- trial1[1,"imgname"]
    r1 <- trial1[1, "studyphase.x"]
    
   # mainDir <- ('C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/OutputFolder2')
  mainDir <- ('C:/Users/kgeier/Desktop/MACHINE_LEARNING_codeForPradeep/OutputFolder2')
    subDira <- paste0("IID_SID_", s1)
    subDird2 <- paste0("repnum_", r1)
    
    
    #Slowly building directory structure 
    path <- file.path(mainDir, subDira)
    dir.create(path, showWarnings = F)
    path <- file.path(path, subDird2)
    dir.create(path, showWarnings = F)
    
    #selecting only the few columns Pradeep wants for Machine learning
    OneTrial <- trial1[, c("SID","TRIAL_LABEL","CURRENT_FIX_BLINK_AROUND",'CURRENT_FIX_DURATION', "studyphase.x",
                           "CURRENT_FIX_START","CURRENT_FIX_END","CURRENT_FIX_INDEX",'CURRENT_FIX_MSG_TEXT_1','CURRENT_FIX_PUPIL', 
                           "CURRENT_FIX_X", "CURRENT_FIX_Y", 'IP_START_TIME', "IP_END_TIME",
                           'TRIAL_FIXATION_TOTAL',"Response","__Reaction_Time__1", "block.x",'image_filename',
                           "SubsequentAccuracy", "test_confidence.y")]
    
    pathfinal <- file.path(mainDir,subDira,subDird2)
    write.csv(x=OneTrial, file= paste0(pathfinal,"/","fixationreport.csv"), quote = F, sep="\t", row.names = FALSE)
    
  }
}

#Recall####

#saving individual trial to it's own folder.
for (m in 1:y){
  mylist3[[m]] -> alltrials
  for (n in 1:length(unique(alltrials$TRIAL_LABEL))) {
    trial1 <- subset(alltrials, subset = alltrials$TRIAL_LABEL == n)
    s1 <- trial1[1, 'IID_SID']
    sa1 <- trial1[1,"SubsequentAccuracy"]
    
    #mainDir <- ('C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/OutputFolder2')
    mainDir <- ('C:/Users/kgeier/Desktop/MACHINE_LEARNING_codeForPradeep/OutputFolder2')
    
    
    subDira <- paste0("IID_SID_", s1)
    
    
    #Slowly building directory structure 
    path <- file.path(mainDir, subDira)
    dir.create(path, showWarnings = F)

    
    pathfinal <- file.path(mainDir,subDira)
    write.table(x=sa1, file= paste0(pathfinal,"/","subsequentaccuracy.txt"), quote = F, col.names = F, row.names = F, sep="\t")
    
  }
}

##########################################################################################
##########################################################################################
##########################################################################################
##########################################################################################
#SACCADE REPORT####

sdif=0
y =37


mylistnew <- list()
for (i in 1:y){
  i = i + sdif #this is to adjust what subject you want to import
  data_dir = 'C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/saccadereports' 
  filename =  i
  fileextension = ".csv"
  filename = paste0(filename, fileextension)
  filepath = paste(data_dir, '/', filename, sep = ''); filepath
  
  tosubset <- fread(filepath)
  
  newdf <- tosubset
  
  #getting rid of "Trial: " so trial number can be numeric (mainly to match the df.merged_wdp)
  newdf$TRIAL_LABEL <- gsub("Trial: ", "", newdf$TRIAL_LABEL)
  newdf$TRIAL_LABEL <- as.numeric(newdf$TRIAL_LABEL)
  i = i -sdif #(part 2 to not have a list 9 items long for subj 7-9)
  mylistnew[[i]] <- newdf
}

RelCol <- c("IID_SID","SID", "RECORDING_SESSION_LABEL", "X__Reaction_Time__1", "block", "imgname", "studyphase",  "oldnearorfar", "test_confidence.x", "TRIAL_LABEL", "SubsequentAccuracy", "SubsequentMorph", "test_confidence.y", "dprime_nearonly_noguess", "dprime_faronly_noguess")

#Loop to merge d' and hit/miss (and many other rows)
mylist2 <- list(seq(1:y))
j = 1 
for (j in 1:y) {
  mylist2[[j]] <- merge(mylistnew[[j]], df.sorted[ , RelCol, with = F], by = c('RECORDING_SESSION_LABEL', 'TRIAL_LABEL'), all = F)
} 

#2.5. set it to show only the used eye's location
#in loop for all subj
mylist3 <- list(seq(1:y)) 
k = 1
for (k in 1:y) {
  #Setting list number to df named visualizermerged
  mylist2[[k]] ->  visualizerMerged
  #Doing nothing bc not relevant for fixation and saccade reports
  
  visualizerMerged -> mylist3[[k]] 
}

#Final Check to see if anything went wrong with the repnum
check.list <- list(seq(1:y))
p = 1 
for (p in 1:y) {
  mylist3[[p]] -> df.tc
  df.er <- subset(df.tc, subset = df.tc$errorcheck != "good")
  check.list[[p]] <- df.er
} 

repcheckall <- do.call("rbind", check.list)


#3. Loop to save with everything in its own folder by trial...

#Encoding####
#saving individual trial to it's own folder.
for (m in 1:y){
  mylist3[[m]] -> alltrials
  for (n in 1:length(unique(alltrials$TRIAL_LABEL))) {
    trial1 <- subset(alltrials, subset = alltrials$TRIAL_LABEL == n)
    s1 <- trial1[1, 'IID_SID']
    i1 <- trial1[1,"imgname"]
    r1 <- trial1[1, "studyphase.x"]
    
    # mainDir <- ('C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/OutputFolder2')
    mainDir <- ('C:/Users/kgeier/Desktop/MACHINE_LEARNING_codeForPradeep/OutputFolder2')
    subDira <- paste0("IID_SID_", s1)
    subDird2 <- paste0("repnum_", r1)
    
    
    #Slowly building directory structure 
    path <- file.path(mainDir, subDira)
    dir.create(path, showWarnings = F)
    path <- file.path(path, subDird2)
    dir.create(path, showWarnings = F)
    
    #selecting only the few columns Pradeep wants for Machine learning
    OneTrial <- trial1[, c("RECORDING_SESSION_LABEL",'IID_SID',"SID","TRIAL_LABEL", "studyphase.x",
                           "CURRENT_SAC_AMPLITUDE",'CURRENT_SAC_ANGLE',"CURRENT_SAC_AVG_VELOCITY",
                           "CURRENT_SAC_CONTAINS_BLINK","CURRENT_SAC_DIRECTION",'CURRENT_SAC_DURATION',
                           'CURRENT_SAC_END_TIME', "CURRENT_SAC_END_TIME", "CURRENT_SAC_START_X", "CURRENT_SAC_START_Y", 
                           "CURRENT_SAC_END_X", "CURRENT_SAC_END_Y", "CURRENT_SAC_INDEX","CURRENT_SAC_MSG_TEXT_1",
                           "CURRENT_SAC_MSG_TIME_1","CURRENT_SAC_PEAK_VELOCITY","CURRENT_SAC_START_TIME",'IP_START_TIME', "IP_END_TIME",
                           "Response","__Reaction_Time__1", "block.x",'image_filename',
                           "test_confidence.y")]
    
    pathfinal <- file.path(mainDir,subDira,subDird2)
    write.csv(x=OneTrial, file= paste0(pathfinal,"/","saccadereport.csv"), quote = F, sep="\t", row.names = FALSE)
    
  }
}


# data_dir = 'C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/saccadereports' 
# filename =  1
# filename2 =  '1_repnum'
# fileextension = ".csv"
# filename = paste0(filename, fileextension)
# filename2 = paste0(filename2, fileextension)
# filepath = paste(data_dir, '/', filename, sep = ''); filepath
# filepath2 = paste(data_dir, '/', filename2, sep = ''); filepath2
# 
# tosubset <- fread(filepath)
# withrepnum <-fread(filepath2)
# 
# 
# a <- merge(tosubset, withrepnum[ ,c('RECORDING_SESSION_LABEL', 'TRIAL_LABEL',"V29"), with = F], by = c('RECORDING_SESSION_LABEL', 'TRIAL_LABEL'), all = F)
# a$studyphase <- a$V29
# b <- a[,1:26]
# 
# pathfinal = 'C:/Users/kgeier/Google Drive/OlsenLab_cloud/All Experiments/EyeTracking/ObjFamETStudy/MACHINE_LEARNING_codeForPradeep/saccadereports' 
# write.csv(x=b, file= paste0(pathfinal,"/","1.csv"), quote = F, sep="\t", row.names = FALSE)