Model Construction

Show/Hide Code
#-------------#
#----NNet-----#
#-------------#
set.seed(1234)
train_control <- trainControl(method = "cv", number = 10)

set.seed(1234)
nnet_model <- train(good ~ ., 
                    data = train, 
                    method = "nnet", 
                    trControl = train_control)

save(nnet_model, file = "dataset\\model\\nnet.model_kfoldCV.Rdata")

K-fold CV

Show/Hide Code
# Data Import
load("dataset\\wine.data_cleaned.Rdata")
load("dataset\\train.Rdata")
load("dataset\\test.Rdata")

# Function Import
load("dataset\\function\\accu.kappa.plot.Rdata")

# Model import
load("dataset\\model\\nnet.model_kfoldCV.Rdata")

nnet.predictions <- predict(nnet_model, newdata = test)

confusionMatrix(nnet.predictions, test$good)
Confusion Matrix and Statistics

          Reference
Prediction   0   1
         0 890 157
         1  59  82
                                          
               Accuracy : 0.8182          
                 95% CI : (0.7951, 0.8397)
    No Information Rate : 0.7988          
    P-Value [Acc > NIR] : 0.0504          
                                          
                  Kappa : 0.3318          
                                          
 Mcnemar's Test P-Value : 4.111e-11       
                                          
            Sensitivity : 0.9378          
            Specificity : 0.3431          
         Pos Pred Value : 0.8500          
         Neg Pred Value : 0.5816          
             Prevalence : 0.7988          
         Detection Rate : 0.7492          
   Detection Prevalence : 0.8813          
      Balanced Accuracy : 0.6405          
                                          
       'Positive' Class : 0               
                                          
Show/Hide Code
nnet.predictions <- as.numeric(nnet.predictions)
pred_obj <- prediction(nnet.predictions, test$good)
auc_val <- performance(pred_obj, "auc")@y.values[[1]]
auc_val
[1] 0.6404628
Show/Hide Code
roc_obj <- performance(pred_obj, "tpr", "fpr")
plot(roc_obj, colorize = TRUE, lwd = 2,
     xlab = "False Positive Rate", 
     ylab = "True Positive Rate",
     main = "Neural Network (10-fold CV)")
abline(a = 0, b = 1)
x_values <- as.numeric(unlist(roc_obj@x.values))
y_values <- as.numeric(unlist(roc_obj@y.values))
polygon(x = x_values, y = y_values, 
        col = rgb(0.3803922, 0.6862745, 0.9372549, alpha = 0.3),
        border = NA)
polygon(x = c(0, 1, 1), y = c(0, 0, 1), 
        col = rgb(0.3803922, 0.6862745, 0.9372549, alpha = 0.3),
        border = NA)
text(0.6, 0.4, paste("AUC =", round(auc_val, 4)))
Show/Hide Code
nnet.kfoldCV.ROC.plot <- recordPlot()

pander::pander(nnet_model$results)
size decay Accuracy Kappa AccuracySD KappaSD
1 0 0.7934 0.1102 0.01513 0.1776
1 1e-04 0.7854 0.02762 0.003511 0.08735
1 0.1 0.7927 0.149 0.01041 0.1597
3 0 0.7905 0.1011 0.0108 0.1638
3 1e-04 0.7963 0.1523 0.0159 0.1971
3 0.1 0.8046 0.323 0.01708 0.0539
5 0 0.7955 0.1138 0.0249 0.1934
5 1e-04 0.7912 0.09769 0.01428 0.1613
5 0.1 0.8071 0.3337 0.016 0.05427

Summary

Show/Hide Code
cowplot::plot_grid(nnet.kfoldCV.ROC.plot)

Model Error Rate Sensitivity Specificity AUC
Neural Network 0.1818 0.9378 0.3431 0.6404628