library(ISLR)


Credit = Default

formel = "balance ~ 1 + as.factor(student) + as.factor(default) + income"

fit.lm = lm(formel,data=Credit)
summary(fit.lm)


#In-sample prediction error
y.pred = predict(fit.lm,Credit)
RMSE.insamp = sqrt(mean((Credit$balance-y.pred)^2))

set.seed(2)
n = nrow(Credit)
ind = sample(1:n,n/2,replace=FALSE)
fit = lm(formel,data=Credit[ind,])
pred = predict(fit,Credit[-ind,])
sqrt(mean((Credit$balance[-ind]-pred)^2))

#Splitting into train and test, repeating several times
n = nrow(Credit)
B = 20
RMSE.train.test = rep(NA,B)
for(b in 1:B)
{
  ind = sample(1:n,n/2,replace=FALSE)
  fit = lm(formel,data=Credit[ind,])
  pred = predict(fit,Credit[-ind,])
  RMSE.train.test[b] = sqrt(mean((Credit$balance[-ind]-pred)^2))
}
RMSE.train.test
#too few data 


plot(rep(mean(RMSE.train.test),20),type = "l",ylim=c(mean(RMSE.train.test)-3,mean(RMSE.train.test)+3))
lines(rep(RMSE.insamp,B),col = 2)

#LOO-CV
n = nrow(Credit)
y.cv = rep(NA,n)
for(i in 1:n)
{
  f = lm(formel,data=Credit[-i,])
  y.cv[i] = predict(f,Credit[i,])
}
RMSE.loo = sqrt(mean((Credit$balance-y.cv)^2))

lines(rep(RMSE.loo,B),col = 3)

#K-fold CV
K = 10
nk = n/K
RMSE.Kcv = rep(0,B)
for(b in 1:B)
{
  pred.K = rep(NA,n)
  ind = sample(1:n,n,replace=FALSE)   #Permutation of data
  for(k in 1:K)
  {
    ind2 = ind[(k-1)*nk + 1:nk]
    f = lm(formel,data=Credit[-ind2,])
    pred.K = predict(f,Credit[ind2,])
    RMSE.Kcv[b] = RMSE.Kcv[b] + sum((Credit$balance[ind2]-pred.K)^2)/n
  }
}
RMSE.Kcv = sqrt(RMSE.Kcv)
lines(rep(mean(RMSE.Kcv),B),col = 4)


RMSE = c(RMSE.insamp,RMSE.train.test,RMSE.loo,RMSE.Kcv)
d = data.frame(RMSE=RMSE,method=c("a-Insamp",rep("b-train/test",B),"c-LOO",rep("d-10-fold",B)))
boxplot(RMSE~method,data=d)