InitialCommit

Models/labels.txt

@@ -0,0 +1,24 @@
+0 A
+1 B
+2 C
+3 D
+4 E
+5 F
+6 G
+7 H
+8 I
+9 K
+10 L
+11 M
+12 N
+13 O
+14 P
+15 Q
+16 R
+17 S
+18 T
+19 U
+20 V
+21 W
+22 X
+23 Y

dataCollection.py

@@ -0,0 +1,62 @@
+import cv2
+from cvzone.HandTrackingModule import HandDetector
+import numpy as np
+import math
+import time
+
+cap = cv2.VideoCapture(0)
+
+detector = HandDetector(maxHands=1)
+offset = 20
+imgSize = 300
+folder = "Data/extra"
+counter = 0
+
+while True:
+    success, img = cap.read()
+    hands, img = detector.findHands(img)
+    if hands:
+
+        hand = hands[0]
+        x, y, w, h = hand['bbox']
+
+        imgWhite = np.ones((imgSize, imgSize, 3), np.uint8)*255
+        imgCrop = img[y - offset:y + h + offset, x - offset:x + w + offset]
+
+        imgCropShape = imgCrop.shape
+
+        aspectRatio = h/w
+
+        imgH, imgW, imgC = imgCrop.shape
+        if imgH > 0 and imgW > 0 and imgC > 0:
+            if aspectRatio > 1:
+                k = imgSize/h
+                wCal = math.ceil(k*w)
+
+                imgResize = cv2.resize(imgCrop,(wCal, imgSize))
+                imgResizeShape = imgResize.shape
+                wGap = math.ceil((imgSize-wCal)/2)
+                imgWhite[:, wGap:wCal+wGap] = imgResize
+            else:
+                k = imgSize / w
+                hCal = math.ceil(k * h)
+                imgResize = cv2.resize(imgCrop, (imgSize,hCal))
+                imgResizeShape = imgResize.shape
+                hGap = math.ceil((imgSize - hCal) / 2)
+                imgWhite[hGap:hCal + hGap,:] = imgResize
+
+            cv2.imshow("ImageCrop", imgCrop)
+            cv2.imshow("ImageWhite", imgWhite)
+
+    cv2.imshow("Image",img)
+    key = cv2.waitKey(1)
+    if key == ord("s"):
+        counter += 1
+
+        cv2.imwrite(f'{folder}/Image_{time.time()}.jpg', imgWhite)
+        #    raise Exception("Could not write image")
+        print(counter)
+
+if cv2.waitKey(10) & 0xFF == ord('q'):
+    cv2.destroyAllWindows()
+

test.py

@@ -0,0 +1,69 @@
+import cv2
+from cvzone.HandTrackingModule import HandDetector
+from cvzone.ClassificationModule import Classifier
+import numpy as np
+import math
+import time
+from pathlib import Path
+
+data_folder = Path("C:/Users/kbinw/Binwant/01 IGDTUW/8th Sem/Final YR Project/Code 4/Sign_lang_project/")
+
+cap = cv2.VideoCapture(0)
+detector = HandDetector(maxHands=1)
+trained_model = Classifier(data_folder/"Models/InceptionV2Model.h5")
+
+offset = 20
+imgSize = 300
+folder = "Data"
+counter = 0
+labels = ['A','B','C','D','E','F','G','H','I','K', 'L', 'M','N','O','P','Q','R','S','T','U','V','W','X','Y']
+
+while True:
+    success, img = cap.read()
+    imgOutput = img.copy()
+    hands, img = detector.findHands(img)
+    if hands:
+        hand = hands[0]
+        x, y, w, h = hand['bbox']
+
+        imgWhite = np.ones((imgSize, imgSize, 3), np.uint8)*255
+        imgCrop = img[y - offset:y + h + offset, x - offset:x + w + offset]
+
+        imgCropShape = imgCrop.shape
+
+        aspectRatio = h/w
+        imgH, imgW, imgC = imgCrop.shape
+        if imgH > 0 and imgW > 0 and imgC > 0:
+            if aspectRatio > 1:
+                k = imgSize/h
+                wCal = math.ceil(k*w)
+                imgResize = cv2.resize(imgCrop,(wCal, imgSize))
+                imgResizeShape = imgResize.shape
+                wGap = math.ceil((imgSize-wCal)/2)
+                imgWhite[:, wGap:wCal+wGap] = imgResize
+                prediction, index = trained_model.getPrediction(imgWhite, draw=True)
+                print(prediction, index)
+
+            else:
+                k = imgSize / w
+                hCal = math.ceil(k * h)
+                imgResize = cv2.resize(imgCrop, (imgSize,hCal))
+                imgResizeShape = imgResize.shape
+                hGap = math.ceil((imgSize - hCal) / 2)
+                imgWhite[hGap:hCal + hGap, :] = imgResize
+                prediction, index = trained_model.getPrediction(imgWhite, draw=False)
+
+            #cv2.rectangle(imgOutput, (x - offset+90, y - offset-50), (x-offset+50, y - offset-50+50), (255, 0, 255), cv2.FILLED)
+
+            cv2.putText(imgOutput, labels[index], (x,y-26), cv2.FONT_HERSHEY_COMPLEX,2,(255,0,255), 2)
+            cv2.rectangle(imgOutput, (x-offset, y-offset), (x + w + offset, y + h + offset), (255, 0, 255), 4)
+
+
+            cv2.imshow("ImageWhite", imgWhite)
+
+    cv2.imshow("Image", imgOutput)
+    cv2.waitKey(1)
+
+
+
+

train.py

@@ -0,0 +1,136 @@
+import random
+import pandas as pd
+import numpy as np
+import os
+import tensorflow
+import cv2
+from tensorflow import keras
+from keras.layers import Input, Dense, Dropout, BatchNormalization, Flatten, Activation, GlobalAveragePooling2D,Conv2D, MaxPooling2D
+from keras.callbacks import EarlyStopping, ModelCheckpoint
+from keras.metrics import categorical_crossentropy
+from keras.preprocessing.image import ImageDataGenerator
+from keras.applications import MobileNet
+from keras.applications import InceptionResNetV2
+from keras.applications.mobilenet import preprocess_input
+from keras.models import Sequential, Model
+from pathlib import Path
+import matplotlib.pyplot as plt
+
+data_folder = Path("C:/Users/kbinw/Binwant/01 IGDTUW/8th Sem/Final YR Project/Code 4/Sign_lang_project/")
+
+img_width, img_height = 224, 224
+img_folder = data_folder/"Data"
+
+n_epoch = 20
+batch_sz = 16
+input_shape = (img_width, img_height, 3)
+input_tensor = Input(shape=(224, 224, 3))
+
+# imports the mobilenet model and discards the last 1000 neuron layer
+base_model = InceptionResNetV2(input_tensor=input_tensor, weights='imagenet', include_top=False)
+
+model = Sequential()
+model.add(base_model)
+model.add(Dropout(0.5))
+model.add(Flatten())
+model.add(BatchNormalization())
+model.add(Dense(1024,kernel_initializer='he_uniform'))
+model.add(BatchNormalization())
+model.add(Activation('relu'))
+model.add(Dropout(0.5))
+model.add(Dense(1024,kernel_initializer='he_uniform'))
+model.add(BatchNormalization())
+model.add(Activation('relu'))
+model.add(Dropout(0.5))
+model.add(Dense(1024,kernel_initializer='he_uniform'))
+model.add(BatchNormalization())
+model.add(Activation('relu'))
+model.add(Dropout(0.5))
+model.add(Dense(24,activation='softmax'))
+
+for layer in base_model.layers:
+    layer.trainable = False
+
+model.summary()
+
+es = EarlyStopping(monitor='val_loss', mode='min', patience=5 ,
+                   restore_best_weights=True, verbose=1)
+
+model.compile(optimizer='adam',
+              loss='categorical_crossentropy',
+              metrics=['acc', tensorflow.keras.metrics.AUC(name='auc'), tensorflow.keras.metrics.AUC(name='roc')])
+
+train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
+                                   validation_split=0.2,
+                                   featurewise_center=False,  # set input mean to 0 over the dataset
+                                   samplewise_center=False,  # set each sample mean to 0
+                                   featurewise_std_normalization=False,  # divide inputs by std of the dataset
+                                   samplewise_std_normalization=False,  # divide each input by its std
+                                   zca_whitening=False,  # apply ZCA whitening
+                                   rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)
+                                   zoom_range=0.1,  # Randomly zoom image
+                                   width_shift_range=0.1,
+                                   # randomly shift images horizontally (fraction of total width)
+                                   height_shift_range=0.1,
+                                   # randomly shift images vertically (fraction of total height)
+                                   horizontal_flip=False,  # randomly flip images
+                                   vertical_flip=False)
+
+train_generator = train_datagen.flow_from_directory(img_folder,
+                                                    target_size=(224,224),
+                                                    color_mode='rgb',
+                                                    batch_size=batch_sz,
+                                                    class_mode='categorical',
+                                                    shuffle=True,
+                                                    subset='training')
+
+validation_generator = train_datagen.flow_from_directory(img_folder,
+                                                         target_size=(224, 224),
+                                                         color_mode='rgb',
+                                                         batch_size=batch_sz,
+                                                         class_mode='categorical',
+                                                         shuffle=False,
+                                                         subset='validation')
+
+step_size_train = train_generator.n//train_generator.batch_size
+
+history = model.fit(train_generator,
+                    validation_data=validation_generator,
+                    validation_steps=validation_generator.samples//validation_generator.batch_size,
+                    steps_per_epoch=step_size_train,
+                    epochs=n_epoch)
+
+# predictions = model.predict_generator(test_generator, test_generator.samples//batch_sz+1)
+# pred = np.argmax(predictions, axis=1)
+# cm = confusion_matrix(test_generator.classes, pred)
+#
+# print('Confusion Matrix')
+# print(cm)
+# print('Classification Report')
+# target_names = ['0', '1', '2', '3', '4']
+# print(classification_report(test_generator.classes, pred, target_names=target_names))
+#
+# disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=target_names)
+# disp.plot(cmap=plt.cm.Blues)
+# plt.show()
+#
+# summarize history for accuracy
+plt.plot(history.history['acc'])
+plt.plot(history.history['val_acc'])
+plt.title('Model Accuracy')
+plt.ylabel('accuracy')
+plt.xlabel('epoch')
+plt.legend(['train', 'val'], loc='upper left')
+plt.show()
+
+# summarize history for loss
+plt.plot(history.history['loss'])
+plt.plot(history.history['val_loss'])
+plt.title('Model Loss')
+plt.ylabel('loss')
+plt.xlabel('epoch')
+plt.legend(['train', 'val'], loc='upper left')
+plt.show()
+
+# Save the trained model for testing and classification in real-time
+model.save(data_folder/"Models/InceptionV2Model.h5")