chr <- read.table("../lrn/cty_chr.txt",
         header=F,colClasses="character",col.names=c("file","feature","type")) 
num <- read.table("../lrn/cty_num.txt",
         header=F,colClasses="character",col.names=c("file","feature","type")) 

target.lrn <- read.table("../lrn/num/472.dat",header=T,colClasses="numeric")
target.val <- read.table("../val/num/472.dat",header=T,colClasses="numeric")

nchr <- length(chr$file)
nnum <- length(num$file)

for (i in 1:nchr) {
  file.lrn <- paste("../lrn/chr/",chr$file[i],".dat",sep="")
  file.val <- paste("../val/chr/",chr$file[i],".dat",sep="")

  print("---------------------------------------------------------------------")
  print(c(chr$feature[i],file.lrn))

  lrn <- read.table(file.lrn,header=T,colClasses="character",blank.lines.skip=F)
  val <- read.table(file.val,header=T,colClasses="character",blank.lines.skip=F)

  y.lrn <- target.lrn[,1]
  y.val <- target.val[,1]
  y     <- c(y.lrn,y.val)

  n.lrn <- length(y.lrn)
  n.val <- length(y.val)
  n     <- length(y)

  wts <- mat.or.vec(n,1) ; for (j in 1:n.lrn) wts[j]=1

  rm(n.lrn,n.val,n)

  x.lrn <- lrn[,1]
  x.val <- val[,1]
  x     <- c(x.lrn,x.val)

  rm(x.lrn, x.val)

  wts <- wts[x!=""]
  y   <- y[x!=""]
  x   <- x[x!=""]

  x <- as.factor(x)

  nl <- nlevels(x)
  print(paste("nlevels = ",nl))

  if (nl == 1) {
    print("too few levels")
  }
  else if (nl < 25) {
    dat <- data.frame(y=y,x=x,w=wts)
    rm(x,y,wts)
    fit <- lm(y~x,data=dat,weights=w)
    #print(anova(fit))
    y.lrn <- dat$y[dat$w==1]
    y.val <- dat$y[dat$w!=1]
    e.lrn <- fit$residuals[dat$w==1] 
    e.val <- fit$residuals[dat$w!=1] 
    n.lrn <- length(e.lrn)
    n.val <- length(e.val)
    R2.lrn <- 1-mean(e.lrn^2)/mean(y.lrn^2)
    R2.val <- 1-mean(e.val^2)/mean(y.val^2)
    print(paste(chr$feature[i],"R2.lrn =",R2.lrn,"n =",n.lrn,"excludes missing"))
    print(paste(chr$feature[i],"R2.val =",R2.val,"n =",n.val,"excludes missing"))
  }
  else if (nl < 100) {
    set.seed(740726)
    idx <- sample(1:length(x),15000)
    dat <- data.frame(y=y[idx],x=x[idx],w=wts[idx])
    rm(x,y,wts,idx)
    fit <- lm(y~x,data=dat,weights=w)
    #print(anova(fit))
    y.lrn <- dat$y[dat$w==1]
    y.val <- dat$y[dat$w!=1]
    e.lrn <- fit$residuals[dat$w==1] 
    e.val <- fit$residuals[dat$w!=1] 
    n.lrn <- length(e.lrn)
    n.val <- length(e.val)
    R2.lrn <- 1-mean(e.lrn^2)/mean(y.lrn^2)
    R2.val <- 1-mean(e.val^2)/mean(y.val^2)
    print(paste(chr$feature[i],"R2.lrn =",R2.lrn,"n =",n.lrn,"excludes missing"))
    print(paste(chr$feature[i],"R2.val =",R2.val,"n =",n.val,"excludes missing"))
  } 
  else {
    print("too many levels")
  }

  print("---------------------------------------------------------------------")
  print(c(chr$feature[i],file.lrn))

  lrn <- read.table(file.lrn,header=T,colClasses="character",blank.lines.skip=F)
  val <- read.table(file.val,header=T,colClasses="character",blank.lines.skip=F)

  y.lrn <- target.lrn[,1]
  y.val <- target.val[,1]
  y     <- c(y.lrn,y.val)

  n.lrn <- length(y.lrn)
  n.val <- length(y.val)
  n     <- length(y)

  wts <- mat.or.vec(n,1) ; for (j in 1:n.lrn) wts[j]=1

  rm(n.lrn,n.val,n)

  x.lrn <- lrn[,1]
  x.val <- val[,1]
  x     <- c(x.lrn,x.val)

  rm(x.lrn, x.val)

  x <- as.factor(x)

  nl <- nlevels(x)
  print(paste("nlevels = ",nl))

  if (nl == 1) {
    print("too few levels")
  }
  else if (nl < 25) {
    dat <- data.frame(y=y,x=x,w=wts)
    rm(x,y,wts)
    fit <- lm(y~x,data=dat,weights=w)
    #print(anova(fit))
    y.lrn <- dat$y[dat$w==1]
    y.val <- dat$y[dat$w!=1]
    e.lrn <- fit$residuals[dat$w==1] 
    e.val <- fit$residuals[dat$w!=1] 
    n.lrn <- length(e.lrn)
    n.val <- length(e.val)
    R2.lrn <- 1-mean(e.lrn^2)/mean(y.lrn^2)
    R2.val <- 1-mean(e.val^2)/mean(y.val^2)
    print(paste(chr$feature[i],"R2.lrn =",R2.lrn,"n =",n.lrn,"includes missing"))
    print(paste(chr$feature[i],"R2.val =",R2.val,"n =",n.val,"includes missing"))
  }
  else if (nl < 100) {
    set.seed(740726)
    idx <- sample(1:length(x),15000)
    dat <- data.frame(y=y[idx],x=x[idx],w=wts[idx])
    rm(x,y,wts,idx)
    fit <- lm(y~x,data=dat,weights=w)
    #print(anova(fit))
    y.lrn <- dat$y[dat$w==1]
    y.val <- dat$y[dat$w!=1]
    e.lrn <- fit$residuals[dat$w==1] 
    e.val <- fit$residuals[dat$w!=1] 
    n.lrn <- length(e.lrn)
    n.val <- length(e.val)
    R2.lrn <- 1-mean(e.lrn^2)/mean(y.lrn^2)
    R2.val <- 1-mean(e.val^2)/mean(y.val^2)
    print(paste(chr$feature[i],"R2.lrn =",R2.lrn,"n =",n.lrn,"includes missing"))
    print(paste(chr$feature[i],"R2.val =",R2.val,"n =",n.val,"includes missing"))
  } 
  else {
    print("too many levels")
  }
}

for (i in 1:nnum) {
  file.lrn <- paste("../lrn/num/",num$file[i],".dat",sep="")
  file.val <- paste("../val/num/",num$file[i],".dat",sep="")

  print("---------------------------------------------------------------------")
  print(c(num$feature[i],file.lrn))

  lrn <- read.table(file.lrn,header=T,colClasses="numeric",blank.lines.skip=F)
  val <- read.table(file.val,header=T,colClasses="numeric",blank.lines.skip=F)

  y.lrn <- target.lrn[,1]
  y.val <- target.val[,1]
  y     <- c(y.lrn,y.val)

  n.lrn <- length(y.lrn)
  n.val <- length(y.val)
  n     <- length(y)
  
  wts <- mat.or.vec(n,1) ; for (j in 1:n.lrn) wts[j]=1

  rm(n.lrn,n.val,n)

  x.lrn <- lrn[,1]
  x.val <- val[,1]
  x     <- c(x.lrn,x.val)

  rm(x.lrn, x.val)
  
  wts <- wts[!is.na(x)]
  y   <- y[!is.na(x)]
  x   <- x[!is.na(x)]

  dat <- data.frame(y=y,x=x,w=wts)
  rm(x,y,wts)
  fit <- lm(y~x,data=dat,weights=w)
  #print(anova(fit))
  y.lrn <- dat$y[dat$w==1]
  y.val <- dat$y[dat$w!=1]
  e.lrn <- fit$residuals[dat$w==1] 
  e.val <- fit$residuals[dat$w!=1] 
  n.lrn <- length(e.lrn)
  n.val <- length(e.val)
  R2.lrn <- 1-mean(e.lrn^2)/mean(y.lrn^2)
  R2.val <- 1-mean(e.val^2)/mean(y.val^2)
  print(paste(num$feature[i],"R2.lrn =",R2.lrn,"n =",n.lrn,"excludes missing"))
  print(paste(num$feature[i],"R2.val =",R2.val,"n =",n.val,"excludes missing"))
}