# install.packages('class')
# use the R routine to perform kNN form the library 'class'
library(class)

# generate training data
set.seed(20200831)
X <- matrix(rnorm(25e4), ncol = 5000)
y <- rbinom(n = 50, size = 1, prob = 0.5)


# CV error with 5-fold cross-validation
cv.error <- 0
for(i in 1:50)
{
  # generate the training data
  set.seed(i)
  X <- matrix(rnorm(25e4), ncol = 5000)
  y <- rbinom(n = 50, size = 1, prob = 0.5)

  # select the 100 features most correlated with the outcome
  univariate.correlations <- apply(X, 2, function(x,y) cor(x,y), y=y)
  best100 <- order(abs(univariate.correlations), decreasing = TRUE)[1:100]
  X.red <- X[, best100]
  
  # 5-fold cross-validation
  set.seed(i)
  index <- sample(rep(1:5, 10))
  cv.err <- cv.err.red <- NULL
  for (k in 1:5)
    cv.err[k] <- sum(y[index == k] != knn1(train = X.red[index != k, ],
                                                test = X.red[index == k, ],
                                                cl = y[index != k]))
  cv.error <- cv.error + sum(cv.err) / nrow(X)

  print(i)
}
# average over 50 simulations
cv.error / 50






###################
### CORRECT WAY ###
###################

# CV error with 5-fold cross-validation
cv.error.val <- 0
for(i in 1:50)
{
  # generate the training data
  set.seed(2 * i)
  X <- matrix(rnorm(25e4), ncol = 5000)
  y <- rbinom(n = 50, size = 1, prob = 0.5)
  
  # 5-fold cross-validation
  index <- sample(rep(1:5, 10))

  cv.err.val <- NULL
  for (k in 1:5)
  {
    # select the 100 features most correlated with the outcome
    univariate.correlations <- apply(X[index != k, ], 2, function(x,y) cor(x,y), y = y[index != k])
    best100 <- order(abs(univariate.correlations), decreasing = TRUE)[1:100]
    X.red <- X[index != k, best100]
    
    cv.err.val[k] <- sum(y[index == k] != knn1(train = X.red,
                                               test = X[index == k, best100],
                                               cl = y[index != k]))
  }
  cv.error.val <- cv.error.val + sum(cv.err.val) / nrow(X)
  print(i)
}

# average over 50 simulations
cv.error.val / 50