if (!require(caret)) install.packages('caret')
library(caret)data <- read.table("../letterdata.csv", sep=",", header = TRUE)
datacreate training and test dataset
set.seed(42)
# Step 1: Get row numbers for the training data
trainRowNumbers <- createDataPartition(data$letter, p=0.8, list=FALSE)
# Step 2: Create the training dataset
trainData <- data[trainRowNumbers,]
# Step 3: Create the test dataset
testData <- data[-trainRowNumbers,]
# Store X and Y for later use.
x = trainData[, 2:17]
y = trainData$letter Check if missing values are in the data
anyNA(trainData)[1] FALSEfitControl <- trainControl(method="cv", number = 10)
metric <- "Accuracy"first_model: treebag
set.seed(42)
model_treebag = train(letter ~ ., data = trainData, method='treebag', trControl = fitControl)
model_treebagBagged CART
16012 samples
16 predictor
26 classes: 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'
No pre-processing
Resampling: Cross-Validated (10 fold)
Summary of sample sizes: 14413, 14408, 14413, 14413, 14409, 14408, ...
Resampling results:
Accuracy Kappa
0.9358616 0.933293second_model: svm
svm3 <- train(letter ~., data = trainData, method = "svmRadial", trControl = fitControl, metric=metric, tuneLength = 10)
svm3Support Vector Machines with Radial Basis Function Kernel
16012 samples
16 predictor
26 classes: 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'
No pre-processing
Resampling: Cross-Validated (10 fold)
Summary of sample sizes: 14411, 14409, 14409, 14410, 14410, 14414, ...
Resampling results across tuning parameters:
C Accuracy Kappa
0.25 0.8794038 0.8745715
0.50 0.9091946 0.9055574
1.00 0.9324284 0.9297223
2.00 0.9471680 0.9450524
4.00 0.9563479 0.9545998
8.00 0.9642800 0.9628497
16.00 0.9681511 0.9668757
32.00 0.9682127 0.9669398
64.00 0.9685879 0.9673301
128.00 0.9679633 0.9666805
Tuning parameter 'sigma' was held constant at a value of 0.0476872
Accuracy was used to select the optimal model using the largest value.
The final values used for the model were sigma = 0.0476872 and C = 64.final model: knn
# prepare parameters for data transform
set.seed(42)
datasetNoMissing <- data[complete.cases(data),]
CX <- datasetNoMissing[,2:17]
preprocessParams <- preProcess(CX, method=c("BoxCox"))
CX <- predict(preprocessParams, CX)
CX# prepare the validation dataset
set.seed(7)
# remove missing values (not allowed in this implementation of knn)
testData <- testData[complete.cases(testData),]
# convert to numeric
for(i in 2:17) {
testData[,i] <- as.numeric(as.character(testData[,i]))
}
# transform the validation dataset
testDataX <- predict(preprocessParams, testData[,2:17])
make predictions
# make predictions
set.seed(7)
predictions <- knn3Train(CX, testDataX, datasetNoMissing$letter, k=3, prob=FALSE)
predictions <- as.factor(predictions)
confusionMatrix(predictions, as.factor(testData$letter))Confusion Matrix and Statistics
Reference
Prediction A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
A 157 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0
B 0 149 0 1 0 0 1 2 0 0 0 0 0 0 0 0 0 2 1 1 0 0 0 0 1 0
C 0 0 145 0 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0
D 0 0 0 160 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
E 0 0 0 0 150 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
F 0 0 0 0 0 152 0 0 0 0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0
G 0 0 1 0 0 0 150 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0
H 0 1 0 0 0 0 0 140 0 0 1 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0
I 0 0 0 0 0 0 0 0 146 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
J 0 0 0 0 0 0 0 0 5 147 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
K 0 0 0 0 0 0 1 1 0 0 141 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0
L 0 0 0 0 1 0 0 0 0 0 0 150 0 0 0 1 0 0 0 0 0 0 0 0 0 0
M 0 0 0 0 0 0 0 0 0 0 0 0 157 0 0 0 0 0 0 0 0 1 1 0 0 0
N 0 0 0 0 0 0 0 0 0 0 0 0 0 155 0 0 0 1 0 0 0 0 0 0 0 0
O 0 0 1 0 0 0 0 2 0 0 0 0 0 1 148 0 1 0 0 0 0 0 0 0 0 0
P 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 157 0 0 0 0 0 0 0 0 0 0
Q 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 155 0 0 0 0 0 0 0 0 1
R 0 1 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 146 0 0 0 0 1 0 0 0
S 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 148 0 0 0 0 1 0 0
T 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 158 0 0 0 0 2 0
U 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 160 0 0 0 0 0
V 0 2 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 151 0 0 0 0
W 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 148 0 0 0
X 0 0 0 0 0 1 0 0 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 155 0 0
Y 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 154 0
Z 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 145
Overall Statistics
Accuracy : 0.984
95% CI : (0.9796, 0.9876)
No Information Rate : 0.0406
P-Value [Acc > NIR] : < 2.2e-16
Kappa : 0.9833
Mcnemar's Test P-Value : NA
Statistics by Class:
Class: A Class: B Class: C Class: D Class: E Class: F Class: G Class: H Class: I Class: J Class: K Class: L Class: M Class: N Class: O Class: P
Sensitivity 1.00000 0.97386 0.98639 0.99379 0.98039 0.98065 0.97403 0.95890 0.96689 0.98658 0.95918 0.98684 0.99367 0.99359 0.98667 0.98125
Specificity 0.99974 0.99765 0.99922 0.99974 1.00000 0.99948 0.99948 0.99896 0.99948 0.99870 0.99896 0.99948 0.99948 0.99974 0.99870 0.99974
Pos Pred Value 0.99367 0.94304 0.97973 0.99379 1.00000 0.98701 0.98684 0.97222 0.98649 0.96711 0.97241 0.98684 0.98742 0.99359 0.96732 0.99367
Neg Pred Value 1.00000 0.99896 0.99948 0.99974 0.99922 0.99922 0.99896 0.99844 0.99870 0.99948 0.99844 0.99948 0.99974 0.99974 0.99948 0.99922
Prevalence 0.03937 0.03837 0.03686 0.04037 0.03837 0.03887 0.03862 0.03661 0.03786 0.03736 0.03686 0.03811 0.03962 0.03912 0.03761 0.04012
Detection Rate 0.03937 0.03736 0.03636 0.04012 0.03761 0.03811 0.03761 0.03511 0.03661 0.03686 0.03536 0.03761 0.03937 0.03887 0.03711 0.03937
Detection Prevalence 0.03962 0.03962 0.03711 0.04037 0.03761 0.03862 0.03811 0.03611 0.03711 0.03811 0.03636 0.03811 0.03987 0.03912 0.03837 0.03962
Balanced Accuracy 0.99987 0.98575 0.99281 0.99676 0.99020 0.99006 0.98675 0.97893 0.98318 0.99264 0.97907 0.99316 0.99657 0.99666 0.99268 0.99049
Class: Q Class: R Class: S Class: T Class: U Class: V Class: W Class: X Class: Y Class: Z
Sensitivity 0.99359 0.96689 0.99329 0.99371 0.98765 0.99342 0.98667 0.98726 0.98089 0.99315
Specificity 0.99922 0.99870 0.99974 0.99948 1.00000 0.99922 0.99974 0.99922 0.99974 0.99974
Pos Pred Value 0.98101 0.96689 0.99329 0.98750 1.00000 0.98052 0.99329 0.98101 0.99355 0.99315
Neg Pred Value 0.99974 0.99870 0.99974 0.99974 0.99948 0.99974 0.99948 0.99948 0.99922 0.99974
Prevalence 0.03912 0.03786 0.03736 0.03987 0.04062 0.03811 0.03761 0.03937 0.03937 0.03661
Detection Rate 0.03887 0.03661 0.03711 0.03962 0.04012 0.03786 0.03711 0.03887 0.03862 0.03636
Detection Prevalence 0.03962 0.03786 0.03736 0.04012 0.04012 0.03862 0.03736 0.03962 0.03887 0.03661
Balanced Accuracy 0.99640 0.98279 0.99651 0.99659 0.99383 0.99632 0.99320 0.99324 0.99032 0.99645| func | Accuracy | Kappa | performance |
|---|---|---|---|
| treebag | 0.9358616 | 0.933293 | normal |
| svm | 0.9682127 | 0.9669398 | good |
| knn | 0.984 | 0.9833 | bes |