{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"import glob\nimport numpy as np\nimport pandas as pd\nimport os\nimport shutil \nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\nimport cv2\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\n%matplotlib inline","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-26T14:15:36.278264Z","iopub.execute_input":"2023-10-26T14:15:36.279394Z","iopub.status.idle":"2023-10-26T14:15:41.60202Z","shell.execute_reply.started":"2023-10-26T14:15:36.279279Z","shell.execute_reply":"2023-10-26T14:15:41.60091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Useful constants","metadata":{}},{"cell_type":"code","source":"images=[]\nlabels=[]\nfeature_dictionary = {\n    'label': tf.io.FixedLenFeature([], tf.int64),\n    'label_normal': tf.io.FixedLenFeature([], tf.int64),\n    'image': tf.io.FixedLenFeature([], tf.string)\n    }\nImage_height = 224\nImage_width = 224","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:15:41.603928Z","iopub.execute_input":"2023-10-26T14:15:41.604518Z","iopub.status.idle":"2023-10-26T14:15:41.610759Z","shell.execute_reply.started":"2023-10-26T14:15:41.604485Z","shell.execute_reply":"2023-10-26T14:15:41.60976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameters","metadata":{}},{"cell_type":"code","source":"def _parse_function(example, feature_dictionary=feature_dictionary):\n    parsed_example = tf.io.parse_example(example, feature_dictionary)\n    return parsed_example\n \ndef read_data(filename):\n    full_dataset = tf.data.TFRecordDataset(filename,num_parallel_reads=tf.data.experimental.AUTOTUNE)\n    print(full_dataset)\n    full_dataset = full_dataset.shuffle(buffer_size=31000)\n    full_dataset = full_dataset.cache()\n    print(\"Size of Training Dataset: \", len(list(full_dataset)))\n    \n    feature_dictionary = {\n    'label': tf.io.FixedLenFeature([], tf.int64),\n    'label_normal': tf.io.FixedLenFeature([], tf.int64),\n    'image': tf.io.FixedLenFeature([], tf.string)\n    }   \n \n    full_dataset = full_dataset.map(_parse_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    print(full_dataset)\n    for image_features in full_dataset:\n        image = image_features['image'].numpy()\n        image = tf.io.decode_raw(image_features['image'], tf.uint8)\n        image = tf.reshape(image, [299, 299])        \n        image=image.numpy()\n        image=cv2.resize(image,(Image_height,Image_width))\n        image=cv2.merge([image,image,image])        \n        #plt.imshow(image)\n        images.append(image)\n        labels.append(image_features['label_normal'].numpy())","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:15:41.612127Z","iopub.execute_input":"2023-10-26T14:15:41.612434Z","iopub.status.idle":"2023-10-26T14:15:41.627858Z","shell.execute_reply.started":"2023-10-26T14:15:41.612407Z","shell.execute_reply":"2023-10-26T14:15:41.626963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"filenames=[\n    '/kaggle/input/ddsm-mammography/training10_0/training10_0.tfrecords',\n    '/kaggle/input/ddsm-mammography/training10_1/training10_1.tfrecords',\n    '/kaggle/input/ddsm-mammography/training10_2/training10_2.tfrecords',\n#     '/kaggle/input/ddsm-mammography/training10_3/training10_3.tfrecords',\n#     '/kaggle/input/ddsm-mammography/training10_4/training10_4.tfrecords'\n          ]\n \nfor file in filenames:\n    read_data(file)\n \nprint(len(images))","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:15:41.630177Z","iopub.execute_input":"2023-10-26T14:15:41.630479Z","iopub.status.idle":"2023-10-26T14:16:54.074563Z","shell.execute_reply.started":"2023-10-26T14:15:41.630452Z","shell.execute_reply":"2023-10-26T14:16:54.073412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"images = np.array(images)\nlabels = np.array(labels)\n \n     \n\nplt.imshow(images[19])\nprint(images[0].shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:16:54.07589Z","iopub.execute_input":"2023-10-26T14:16:54.076217Z","iopub.status.idle":"2023-10-26T14:16:55.862762Z","shell.execute_reply.started":"2023-10-26T14:16:54.076187Z","shell.execute_reply":"2023-10-26T14:16:55.861803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(images, labels, test_size=0.20, stratify = labels, random_state=10)\nx_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.50, stratify = y_test, random_state=50)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:16:55.863983Z","iopub.execute_input":"2023-10-26T14:16:55.864293Z","iopub.status.idle":"2023-10-26T14:16:58.070062Z","shell.execute_reply.started":"2023-10-26T14:16:55.864264Z","shell.execute_reply":"2023-10-26T14:16:58.068883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications.vgg16 import VGG16\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, Conv2D, multiply, LocallyConnected2D, Lambda\nfrom keras.models import Model\nin_lay = Input(x_train.shape[1:])\nbase_pretrained_model = VGG16(input_shape = x_train.shape[1:], include_top = False, weights = 'imagenet')\nbase_pretrained_model.trainable = False\npt_features = base_pretrained_model(in_lay)\npt_depth = base_pretrained_model.get_output_shape_at(0)[-1]\npt_features = base_pretrained_model(in_lay)\nfrom keras.layers import BatchNormalization\nbn_features = BatchNormalization()(pt_features)\n# here we do an attention mechanism to turn pixels in the GAP on an off\nattn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(bn_features)\nattn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = LocallyConnected2D(1, \n                                kernel_size = (1,1), \n                                padding = 'valid', \n                                activation = 'sigmoid')(attn_layer)\n                        \n# fan it out to all of the channels\nup_c2_w = np.ones((1, 1, 1, pt_depth))\nup_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', \n               activation = 'linear', use_bias = False, weights = [up_c2_w])\nup_c2.trainable = False\nattn_layer = up_c2(attn_layer)\n\nmask_features = multiply([attn_layer, bn_features])\ngap_features = GlobalAveragePooling2D()(mask_features)\ngap_mask = GlobalAveragePooling2D()(attn_layer)\n\n# to account for missing values from the attention model\ngap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\ngap_dr = Dropout(0.5)(gap)\ndr_steps = Dropout(0.25)(Dense(128, activation = 'elu')(gap_dr))\nout_layer = Dense(1, activation = 'sigmoid')(dr_steps)\n\nfinal_model = Model(inputs = [in_lay], outputs = [out_layer])\nfinal_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:16:58.071554Z","iopub.execute_input":"2023-10-26T14:16:58.071897Z","iopub.status.idle":"2023-10-26T14:16:59.077083Z","shell.execute_reply.started":"2023-10-26T14:16:58.071866Z","shell.execute_reply":"2023-10-26T14:16:59.075894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('breast_cancer')\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.00001)\n#reduceLROnPlat = ReduceLROnPlateau()\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=5) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]\n     \n\n# Compile the model\nfinal_model.compile(tf.keras.optimizers.Adam(learning_rate=.001), \n              loss='binary_crossentropy', \n              metrics=['acc'])\n     \n\n# train the model\n\nhistory = final_model.fit(x_train, y_train, \n                          steps_per_epoch=len(x_train)//8,\n                          epochs=30,\n                          validation_data = (x_val, y_val),\n                          validation_steps = len(x_val)//8, \n                          callbacks = callbacks_list\n                        )\n\nloss_value , accuracy = final_model.evaluate(x_test, y_test)\n\nprint('Test_loss_value = ' +str(loss_value))\nprint('test_accuracy = ' + str(accuracy))\n     \n\n# PLOTTING RESULTS (Train vs Validation) \nimport matplotlib.pyplot as plt\nimport numpy as np\n\ndef Train_Val_Plot(acc,val_acc,loss,val_loss):\n    \n    fig, (ax1, ax2) = plt.subplots(1,2, figsize= (18,8))\n    fig.suptitle(\" MODEL'S METRICS VISUALIZATION \")\n    ax1.set_xticks(np.arange(1, 21, 1))\n    \n    ax1.plot(range(1, len(acc) + 1), acc)\n    ax1.plot(range(1, len(val_acc) + 1), val_acc)\n    #ax1.set_title('History of Accuracy')\n    ax1.set_xlabel('Epochs')\n    ax1.set_ylabel('Accuracy')\n    ax1.legend(['training', 'validation'])\n    #plt.xticks(np.arange(1, 21, 1))\n    ax2.set_xticks(np.arange(1, 21, 1))\n    ax2.plot(range(1, len(loss) + 1), loss)\n    ax2.plot(range(1, len(val_loss) + 1), val_loss)\n    #ax2.set_title('History of Loss')\n    ax2.set_xlabel('Epochs')\n    ax2.set_ylabel('Loss')\n    ax2.legend(['training', 'validation']) \n    \n    plt.show()\n    fig.savefig('figure.png')\n    \nTrain_Val_Plot(history.history['acc'],history.history['val_acc'],\n               history.history['loss'],history.history['val_loss'])","metadata":{"execution":{"iopub.status.busy":"2023-10-26T14:16:59.078643Z","iopub.execute_input":"2023-10-26T14:16:59.079348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = final_model.layers[-2].output\n\nmodel_feat = keras.Model(inputs = final_model.inputs ,outputs = predictions)\n\nExtracted_features = model_feat.predict(x_test)\nprint(Extracted_features.shape)\n     \n\ntrain_image, valid_image, train_label, valid_label = train_test_split(Extracted_features, y_test, test_size=0.20, stratify = y_test, random_state=10)\nval_img, test_img, val_lab, test_lab = train_test_split(valid_image, valid_label, test_size=0.5, \n                                                        stratify = valid_label, random_state=50)\n     \n\nfrom sklearn.metrics import accuracy_score, recall_score, precision_score\nfrom sklearn.neighbors import KNeighborsClassifier\n_classifier = KNeighborsClassifier(n_neighbors=5)\n     \n\n_classifier.fit(train_image, train_label)\n     \n\npred = _classifier.predict(test_img)\nacc = accuracy_score(y_true = test_lab, y_pred = pred)\nprint(acc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_features = Extracted_features\nlab = np.array(y_test)\nlab = lab.reshape(lab.shape[0], 1)\nprint(final_features.shape)\nprint(lab.shape)\nnp.savetxt(\"ext_features.csv\", final_features, delimiter=\",\")\nnp.savetxt(\"labels.csv\", lab, delimiter=\",\")\n     \n\ndf = pd.read_csv('ext_features.csv', header=None)\ndf2 = pd.read_csv('labels.csv', header=None)\n\nprint(df.shape)\nprint(df2.shape)\ntotal_features=df.shape[1]\nx=df[df.columns[:total_features]]\ny=df2[df2.columns[-1]].astype(int)\n\nprint(x.shape)\nprint(y.shape)\nprint(total_features)\n     \n\nfrom sklearn.model_selection import train_test_split\ntrain_x, test_x, train_y, test_y = train_test_split(x, y, test_size=0.2, stratify = y, random_state=10)\nprint(train_x.shape)\nprint(test_y.shape)\n     ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#adaptivebeta\nimport numpy as np\nimport pandas as pd\nimport math\nimport random\nimport time\nfrom sklearn.metrics import accuracy_score, recall_score, precision_score, classification_report\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.preprocessing import StandardScaler\n\n\n\nswarm_size = 20   #population size\nmax_iterations = 100\nomega = 0.2  #used in the fitness function\ndelta=0.2   #to set an upper limit for including a slightly worse particle in LAHC\n\n\ndef mutate(agent):\n  percent=0.2\n  numChange=int(total_features*percent)\n  pos=np.random.randint(0,total_features-1,numChange) #choose random positions to be mutated\n  agent[pos]=1-agent[pos] #mutation\n  return agent\n\ndef LAHC(particle):\n            _lambda = 15 #upper limit on number of iterations in LAHC\n            target_fitness = find_fitness(particle) #original fitness\n            for i in range(_lambda):\n                    new_particle = mutate(particle) #first mutation\n                    temp = find_fitness(new_particle)\n                    if temp < target_fitness:\n                        particle = new_particle.copy() #updation\n                        target_fitness = temp\n                    elif (temp<=(1+delta)*target_fitness):\n                        temp_particle = new_particle.copy()\n                        for j in range(_lambda):\n                            temp_particle1 = mutate(temp_particle) #second mutation\n                            temp_fitness = find_fitness(temp_particle1)\n                            if temp_fitness < target_fitness:\n                                target_fitness=temp_fitness\n                                particle=temp_particle1.copy() #updation\n                            break\n            return particle   \n\ndef randomwalk(agent):\n    percent = 30\n    percent /= 100\n    neighbor = agent.copy()\n    size = np.shape(agent)[0]\n    upper = int(percent*size)\n    if upper <= 1:\n        upper = size\n    x = random.randint(1,upper)\n    pos = random.sample(range(0,size - 1),x)\n    for i in pos:\n        neighbor[i] = 1 - neighbor[i]\n    return neighbor\n\ndef adaptiveBeta(agent):\n    bmin = 0.1 #parameter: (can be made 0.01)\n    bmax = 1\n    maxIter = 10 # parameter: (can be increased )\n    \n    agentFit = find_fitness(agent)\n    for curr in range(maxIter):\n        neighbor = agent.copy()\n        size = np.shape(neighbor)[0]\n        neighbor = randomwalk(neighbor)\n\n        beta = bmin + (curr / maxIter)*(bmax - bmin)\n        for i in range(size):\n            random.seed( time.time() + i )\n            if random.random() <= beta:\n                neighbor[i] = agent[i]\n        neighFit = find_fitness(neighbor)\n        if neighFit <= agentFit:\n            agent = neighbor.copy()\n            \n\n\n    return agent\n\ndef find_fitness(particle):\n            features = []\n            for x in range(len(particle)):\n                    if particle[x]>=0.5: #convert it to zeros and ones\n                        features.append(df.columns[x])\n            if(len(features)==0):\n                        return 10000\n            new_x_train = train_x[features].copy()\n            new_x_test = test_x[features].copy()\n\n            _classifier = KNeighborsClassifier(n_neighbors=5)\n            _classifier.fit(new_x_train, train_y)\n            predictions = _classifier.predict(new_x_test)\n            acc = accuracy_score(y_true = test_y, y_pred = predictions)\n            fitness = acc\n            err=1-acc\n            num_features = len(features)\n            fitness =  alpha*err + (1-alpha)*(num_features/total_features)\n\n            return fitness\n\ndef transfer_func(velocity): #to convert into an array of zeros and ones\n            t=[]\n            for i in range(len(velocity)):\n                    t.append(abs(velocity[i]/(math.sqrt(1+velocity[i]*velocity[i])))) #transfer function inside paranthesis\n            return t\n\n#initialize swarm position and swarm velocity of SSD\nswarm_vel = np.random.uniform(low=0, high=1, size=(swarm_size,total_features))\n\nswarm_pos = np.random.uniform(size=(swarm_size,total_features))\nswarm_pos = np.where(swarm_pos>=0.5,1,0)\n\nc = 100\nalpha= 0.9\n\ngbest_fitness=100000\npbest_fitness = np.zeros(swarm_size)\npbest_fitness.fill(np.inf)  #initialize with the worse possible values\npbest = np.empty((swarm_size,total_features))\ngbest = np.empty(total_features)\npbest.fill(np.inf)\ngbest.fill(np.inf)\n\nfor itr in range(max_iterations):\n\n                for i in range(swarm_size):\n                  \n                    swarm_pos[i] = adaptiveBeta(swarm_pos[i]) #for ABHC local search\n                    #swarm_pos[i] = LAHC(swarm_pos[i]) #for LAHC local search\n                    fitness = find_fitness(swarm_pos[i])\n\n                    if fitness < gbest_fitness:\n\n                        gbest=swarm_pos[i].copy() #updating global best\n                        gbest_fitness=fitness\n\n\n\n                    if fitness < pbest_fitness[i]:\n                        pbest[i] = swarm_pos[i].copy() #updating personal best\n                        pbest_fitness[i]=fitness\n\n                    r1 = random.random()\n                    r2 = random.random()\n\n          #updating the swarm velocity\n                    if r1 < 0.5:\n                        swarm_vel[i] = c*math.sin(r2)*(pbest[i]-swarm_pos[i]) +math.sin(r2)* (gbest-swarm_pos[i])\n                    else:\n                        swarm_vel[i] = c*math.cos(r2)*(pbest[i]-swarm_pos[i]) + math.cos(r2)*(gbest-swarm_pos[i])\n                    \n          #decaying value of c\n                    alpha= 0.9\n                    c=alpha*c;\n          \n          #applying transfer function and then updating the swarm position\n                    t = transfer_func(swarm_vel[i])\n                    for j in range(len(swarm_pos[i])):\n                        if(t[j] < 0.5):\n                            swarm_pos[i][j] = swarm_pos[i][j]\n                        else:\n                            swarm_pos[i][j] = 1 - swarm_pos[i][j]\n\n\n\nselected_features = gbest\nprint(gbest_fitness)\n            \nnumber_of_selected_features = np.sum(selected_features)\nprint(\"#\",number_of_selected_features)\n\nfeatures=[]\nfor j in range(len(selected_features)):\n                if selected_features[j]==1:\n                        features.append(df.columns[j])\nnew_x_train = train_x[features]\nnew_x_test = test_x[features]\n\n_classifier = KNeighborsClassifier(n_neighbors=5)\n_classifier.fit(new_x_train, train_y)\npredictions = _classifier.predict(new_x_test)\nacc = accuracy_score(y_true = test_y, y_pred = predictions)\npre = precision_score(y_true = test_y, y_pred = predictions,average=None)\nrec = recall_score(y_true = test_y, y_pred = predictions, average=None)\nresult = classification_report(y_true = test_y, y_pred = predictions, digits=5)\nfitness = acc\nprint(\"Acc:\",fitness)\nprint(\"Precision:\", pre)\nprint(\"Recall:\",rec)\nprint(result)\nprint(\"\\n\\n\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Procedure","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, roc_auc_score\nfpr, tpr, thresholds = roc_curve(test_y, predictions)\n\nauc = auc(fpr, tpr)\nsc = roc_auc_score(test_y, predictions)\nprint(auc)\nprint(sc)\n\nimport matplotlib.pyplot as plt\nplt.figure(1)\nplt.plot([0, 1], [0, 1],\"--\")\n\nplt.plot(fpr, tpr, label='ROC curve (area = {:.3f})'.format(auc), color='orange')\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()\n     ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}