{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.5"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing important libraries ","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\n\nimport splitfolders\n\nimport tensorflow as tf\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Conv2D,Dropout,Flatten,MaxPooling2D, BatchNormalization,Input,concatenate\nfrom keras.callbacks import EarlyStopping,ReduceLROnPlateau\nfrom keras.utils import plot_model\n\nfrom sklearn.metrics import classification_report, confusion_matrix","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install split_folders","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing dataset","metadata":{}},{"cell_type":"markdown","source":"For the experiment and testing whether neural networks runs successfully taking max 100 images from each of the folder of 29 folders so we have 2900 images total","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# Path where our data is located\nbase_path = \"../Downloads/asl_alphabet_train/asl_alphabet_train\"\n\n# Dictionary to save our 29 classes\ncategories = {\n    0: \"A\",\n    1: \"B\",\n    2: \"C\",\n    3: \"D\",\n    4: \"E\",\n    5: \"F\",\n    6: \"G\",\n    7: \"H\",\n    8: \"I\",\n    9: \"J\",  # Fixed typo here\n    10: \"K\",\n    11: \"L\",\n    12: \"M\",\n    13: \"N\",\n    14: \"O\",\n    15: \"P\",\n    16: \"Q\",\n    17: \"R\",\n    18: \"S\",\n    19: \"T\",\n    20: \"U\",\n    21: \"V\",\n    22: \"W\",\n    23: \"X\",\n    24: \"Y\",\n    25: \"Z\",\n    26: \"del\",\n    27: \"nothing\",\n    28: \"space\",\n}\nimport os\nimport pandas as pd\n\n# Function to add class name prefix to filenames\ndef add_class_name_prefix(df, column_name):\n    df[column_name] = df.apply(lambda row: str(row['category']) + '_' + str(row[column_name]), axis=1)\n    return df\n\n\n# List containing all the filenames in the dataset\nfilenames_list = []\n# List to store the corresponding category; each folder of the dataset has one class of data\ncategories_list = []\n\n# Maximum number of records to fetch per category\nmax_records_per_category = 3000\n\nfor category in categories:\n    category_path = os.path.join(base_path, categories[category])\n    filenames = os.listdir(category_path)[:max_records_per_category]\n    filenames_list += filenames\n    categories_list += [category] * len(filenames)\n\ndf = pd.DataFrame({\"filename\": filenames_list, \"category\": categories_list})\ndf = add_class_name_prefix(df, \"filename\")\n\n# Shuffle the dataframe\ndf = df.sample(frac=1).reset_index(drop=True)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories[df.category[0]]\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(categories[df.category[0]])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.filename[0]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of elements = \", len(df))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting Bar graph for different sign images ","metadata":{}},{"cell_type":"code","source":"label,count = np.unique(df.category,return_counts=True)\nuni = pd.DataFrame(data=count,index=categories.values(),columns=['Count'])\n\nplt.figure(figsize=(14,4),dpi=200)\nsns.barplot(data=uni,x=uni.index,y='Count',palette='icefire',width=0.4).set_title('Class distribution in Dataset',fontsize=15)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting each sign image samples","metadata":{}},{"cell_type":"code","source":"classes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',\n           'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',\n           'W', 'X', 'Y', 'Z', 'del', 'nothing', 'space']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sample_images():\n    figure = plt.figure()\n    plt.figure(figsize=(16,5))\n\n    for i in range (0,29):\n        plt.subplot(3,10,i+1)\n        plt.xticks([])\n        plt.yticks([])\n        path = base_path + \"/{0}/{0}1.jpg\".format(classes[i])\n        img = plt.imread(path)\n        plt.imshow(img)\n        plt.xlabel(classes[i])\nplot_sample_images()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spliting files","metadata":{}},{"cell_type":"markdown","source":"80% for training, 10% for validation and 10% for testing","metadata":{}},{"cell_type":"code","source":"import os\nimport splitfolders\n\n# Path to the original dataset\ninput_folder = '../Downloads/asl_alphabet_train/asl_alphabet_train'\n\n# Path to the output folder for the split data\noutput_folder = '../Downloads/Google/working'\n\n# Number of data points to take from each subfolder\nnum_samples_per_category = 3000\n\n# Create output folder if it doesn't exist\nos.makedirs(output_folder, exist_ok=True)\n\n# Loop through each subfolder and copy 100 samples\nfor category in os.listdir(input_folder):\n    category_path = os.path.join(input_folder, category)\n    output_category_path = os.path.join(output_folder, category)\n    os.makedirs(output_category_path, exist_ok=True)\n    \n    filenames = os.listdir(category_path)[:num_samples_per_category]\n    \n    for filename in filenames:\n        src_path = os.path.join(category_path, filename)\n        dst_path = os.path.join(output_category_path, filename)\n        os.replace(src_path, dst_path)\n\n# Perform the train-validation-test split\nsplitfolders.ratio(output_folder, output=output_folder, seed=1333, ratio=(0.8, 0.1, 0.1))\n","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Resizing and rescaling images","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(rescale= 1.0 / 255)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '../Downloads/Google/working/train'\nval_path = '../Downloads/Google/working/val'\ntest_path = '../Downloads/Google/working/test'\n\nbatch = 32\nimage_size = 50\nimg_channel = 3\nn_classes = 29","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest'\n)\n\n\n                                      ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = datagen.flow_from_directory(directory=train_path,\n                                         target_size=(image_size, image_size),\n                                         batch_size=batch,\n                                         class_mode='categorical')\n\nval_data = datagen.flow_from_directory(directory=val_path,\n                                       target_size=(image_size, image_size),\n                                       batch_size=batch,\n                                       class_mode='categorical')\n\ntest_data = datagen.flow_from_directory(directory=test_path,\n                                        target_size=(image_size, image_size),\n                                        batch_size=batch,\n                                        class_mode='categorical',\n                                        shuffle=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN model","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n# input layer\n# Block 1\nmodel.add(Conv2D(32,3,activation='relu',padding='same',input_shape = (image_size,image_size,img_channel)))\nmodel.add(Conv2D(32,3,activation='relu',padding='same'))\n#model.add(BatchNormalization())\nmodel.add(MaxPooling2D(padding='same'))\nmodel.add(Dropout(0.2))\n\n# Block 2\nmodel.add(Conv2D(64,3,activation='relu',padding='same'))\nmodel.add(Conv2D(64,3,activation='relu',padding='same'))\n#model.add(BatchNormalization())\nmodel.add(MaxPooling2D(padding='same'))\nmodel.add(Dropout(0.3))\n\n#Block 3\nmodel.add(Conv2D(128,3,activation='relu',padding='same'))\nmodel.add(Conv2D(128,3,activation='relu',padding='same'))\n#model.add(BatchNormalization())\n#model.add(MaxPooling2D(padding='same'))\nmodel.add(Dropout(0.4))\n\n# fully connected layer\nmodel.add(Flatten())\n\nmodel.add(Dense(512,activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(128,activation='relu'))\nmodel.add(Dropout(0.3))\n\n# output layer\nmodel.add(Dense(29, activation='softmax'))\n\n\n\nmodel.summary()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN model architecture","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"PATH\"] += os.pathsep + 'C:/Program Files/Graphviz/bin'  # Adjust the path to your Graphviz installation directory\n\n\nplot_model(model, to_file='model_architecture.png', show_shapes=True, show_layer_names=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install pydot","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, to_file = \"CNN.png\", show_shapes=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example: reduce patience in EarlyStopping and ReduceLROnPlateau\nearly_stopping = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=3, restore_best_weights=True, verbose=0)\nreduce_learning_rate = ReduceLROnPlateau(monitor='val_accuracy', patience=1, factor=0.5, verbose=1)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss = 'categorical_crossentropy' , metrics=['accuracy'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asl_class = model.fit(train_data,\n                      validation_data= val_data,\n                      epochs=10,\n                      callbacks=[early_stopping,reduce_learning_rate],\n                      verbose = 1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nloss, accuracy = model.evaluate(val_data)\nprint(f'Validation Loss: {loss}, Validation Accuracy: {accuracy}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\n\n# Assume you have trained the model and obtained the 'history' object\nhistory = asl_class\n\n# Plot training and validation accuracy\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n# Plot training and validation loss\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# AlexNET","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models\n\n\n# AlexNet model\nmodel1 = models.Sequential()\n\n# Block 1\nmodel1.add(layers.Conv2D(96, (11, 11), strides=(4, 4), activation='relu', input_shape=(image_size,image_size,img_channel)))\nmodel1.add(layers.MaxPooling2D((3, 3), strides=(2, 2)))\n\n# Block 2\nmodel1.add(layers.Conv2D(256, (5, 5), padding='same', activation='relu'))\nmodel1.add(layers.MaxPooling2D((3, 3), strides=(2, 2)))\n\n# Block 3\nmodel1.add(layers.Conv2D(384, (3, 3), padding='same', activation='relu'))\n\n# Block 4\nmodel1.add(layers.Conv2D(384, (3, 3), padding='same', activation='relu'))\n\n# Block 5\nmodel1.add(layers.Conv2D(256, (3, 3), padding='same', activation='relu'))\n#model1.add(layers.MaxPooling2D((3, 3), strides=(2, 2)))\n\n# Flatten the output for fully connected layers\nmodel1.add(layers.Flatten())\n\n# Fully connected layers\nmodel1.add(layers.Dense(4096, activation='relu'))\nmodel1.add(layers.Dropout(0.5))\n\nmodel1.add(layers.Dense(4096, activation='relu'))\nmodel1.add(layers.Dropout(0.5))\n\n# Output layer\nmodel1.add(layers.Dense(n_classes, activation='softmax'))\n\n# Display the model summary\nmodel1.summary()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"PATH\"] += os.pathsep + 'C:/Program Files/Graphviz/bin'  # Adjust the path to your Graphviz installation directory\n\n\nplot_model(model1, to_file='model_architecture1.png', show_shapes=True, show_layer_names=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel1.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nasl_class2 = model1.fit(train_data, validation_data=val_data, epochs=10)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\n\n# Assume you have trained the model and obtained the 'history' object\nhistory = asl_class2\n\n# Plot training and validation accuracy\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n# Plot training and validation loss\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BERT","metadata":{}},{"cell_type":"code","source":"from transformers import BertTokenizer\nfrom keras.layers import Input\nfrom transformers import TFBertForSequenceClassification\n\n# Define input layers for image and text\nimage_input = Input(shape=(image_size, image_size, 3))\ntext_input = Input(shape=(), dtype=tf.string)  # Assuming text_input is a string\n\n# Convert text_input to string\ntext_str = tf.strings.reduce_join(text_input, axis=-1, separator=' ')\n\n# Tokenize text input using BERT tokenizer\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\ntext_tokens = tokenizer(str(text_str), padding=True, truncation=True, return_tensors=\"tf\")['input_ids']\n\n# Load pre-trained BERT model\nbert_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased')\nbert_embeddings = bert_model(text_tokens)[0]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Flatten and concatenate visual and textual features\nvisual_flatten = Flatten()(image_input)\ncombined_features = tf.concat([visual_flatten, bert_embeddings], axis=1)\n\n# Define fully connected layers for classification\nfc1 = Dense(512, activation='relu')(combined_features)\nfc1_dropout = Dropout(0.2)(fc1)\nfc2 = Dense(128, activation='relu')(fc1_dropout)\nfc2_dropout = Dropout(0.3)(fc2)\noutput = Dense(n_classes, activation='softmax')(fc2_dropout)\n\n# Create model\ncombined_model = tf.keras.Model(inputs=[image_input, text_input], outputs=output)\n\n# Compile the model\ncombined_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Print model summary\ncombined_model.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"PATH\"] += os.pathsep + 'C:/Program Files/Graphviz/bin'  # Adjust the path to your Graphviz installation directory\n\n\nplot_model(combined_model, to_file='model_architecture3.png', show_shapes=True, show_layer_names=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nasl_class3 = model.fit(\n    train_data,\n    epochs=10,\n    validation_data=val_data  # Just pass val_data directly\n)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nloss, accuracy = model.evaluate(val_data)\nprint(f'Validation Loss: {loss}, Validation Accuracy: {accuracy}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\n\n# Assume you have trained the model and obtained the 'history' object\nhistory = asl_class3\n\n# Plot training and validation accuracy\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n# Plot training and validation loss\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ST-GCN","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models\n\ndef ST_GCN(input_shape, num_classes):\n    # Define the input layer\n    inputs = layers.Input(shape=input_shape)\n    \n    # Spatial Temporal Graph Convolutional Layers\n    x = layers.Conv2D(filters=64, kernel_size=(1, 3), strides=(1, 1), padding='same')(inputs)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Conv2D(filters=64, kernel_size=(1, 3), strides=(1, 1), padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Conv2D(filters=64, kernel_size=(1, 3), strides=(1, 1), padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.MaxPooling2D(pool_size=(1, 2))(x)\n    \n    x = layers.Conv2D(filters=128, kernel_size=(1, 3), strides=(1, 1), padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Conv2D(filters=128, kernel_size=(1, 3), strides=(1, 1), padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.Conv2D(filters=128, kernel_size=(1, 3), strides=(1, 1), padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.ReLU()(x)\n    x = layers.MaxPooling2D(pool_size=(1, 2))(x)\n    \n    # Flatten the output\n    x = layers.Flatten()(x)\n    \n    # Fully connected layers\n    x = layers.Dense(512, activation='relu')(x)\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(128, activation='relu')(x)\n    x = layers.Dropout(0.5)(x)\n    \n    # Output layer\n    outputs = layers.Dense(num_classes, activation='softmax')(x)\n    \n    # Create the model\n    model = models.Model(inputs=inputs, outputs=outputs, name='ST_GCN')\n    \n    return model\n\n# Example usage:\ninput_shape = (50, 50, 3)  # Input shape should match your image size and channels\nnum_classes = 29  # Number of classes in your ASL dataset\nst_gcn_model = ST_GCN(input_shape, num_classes)\nst_gcn_model.summary()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"PATH\"] += os.pathsep + 'C:/Program Files/Graphviz/bin'  # Adjust the path to your Graphviz installation directory\n\n\nplot_model(st_gcn_model, to_file='model_architecture3.png', show_shapes=True, show_layer_names=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nasl_class4 = model.fit(\n    train_data,\n    epochs=10,\n    validation_data=val_data  # Just pass val_data directly\n)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model\nloss, accuracy = model.evaluate(val_data)\nprint(f'Validation Loss: {loss}, Validation Accuracy: {accuracy}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Assume you have trained the model and obtained the 'history' object\nhistory = asl_class4\n\n# Plot training and validation accuracy\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()\n\n# Plot training and validation loss\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}