{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom tensorflow import keras\nkeras.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport tensorflow as tf\nimport numpy as np\n\nclass QuadraticWeightedKappa(tf.keras.metrics.Metric):\n    def __init__(self, maxClassesCount=6, name='Kappa', **kwargs):        \n        super(QuadraticWeightedKappa, self).__init__(name=name, **kwargs)\n        self.M = maxClassesCount\n\n        self.O = self.add_weight(name='O', initializer='zeros',shape=(self.M,self.M,), dtype=tf.int64)\n        self.W = self.add_weight(name='W', initializer='zeros',shape=(self.M,self.M,), dtype=tf.float32)\n        self.actualHist = self.add_weight(name='actHist', initializer='zeros',shape=(self.M,), dtype=tf.int64)\n        self.predictedHist = self.add_weight(name='predHist', initializer='zeros',shape=(self.M,), dtype=tf.int64)\n        \n        # filling up the content of W once\n        w = np.zeros((self.M,self.M),dtype=np.float32)\n        for i in range(0,self.M):\n            for j in range(0,self.M):\n                w[i,j] = (i-j)*(i-j) / ((self.M - 1)*(self.M - 1))\n        self.W.assign(w)\n    \n    def reset_states(self):\n        \"\"\"Resets all of the metric state variables.\n        This function is called between epochs/steps,\n        when a metric is evaluated during training.\n        \"\"\"\n        # value should be a Numpy array\n        zeros1D = np.zeros(self.M)\n        zeros2D = np.zeros((self.M,self.M))\n        tf.keras.backend.batch_set_value([\n            (self.O, zeros2D),\n            (self.actualHist, zeros1D),\n            (self.predictedHist,zeros1D)\n        ])\n\n\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        # shape is: Batch x 1\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1])\n\n        y_true_int = tf.cast(tf.math.round(y_true), dtype=tf.int64)\n        y_pred_int = tf.cast(tf.math.round(y_pred), dtype=tf.int64)\n\n        confM = tf.math.confusion_matrix(y_true_int, y_pred_int, dtype=tf.int64, num_classes=self.M)\n\n        # incremeting confusion matrix and standalone histograms\n        self.O.assign_add(confM)\n\n        cur_act_hist = tf.math.reduce_sum(confM, 0)\n        self.actualHist.assign_add(cur_act_hist)\n\n        cur_pred_hist = tf.math.reduce_sum(confM, 1)\n        self.predictedHist.assign_add(cur_pred_hist)\n\n    def result(self):\n        EFloat = tf.cast(tf.tensordot(self.actualHist,self.predictedHist, axes=0),dtype=tf.float32)\n        OFloat = tf.cast(self.O,dtype=tf.float32)\n        \n        # E must be normalized \"such that E and O have the same sum\"\n        ENormalizedFloat = EFloat / tf.math.reduce_sum(EFloat) * tf.math.reduce_sum(OFloat)\n\n        \n        return 1.0 - tf.math.reduce_sum(tf.math.multiply(self.W, OFloat))/tf.math.reduce_sum(tf.multiply(self.W, ENormalizedFloat))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\n#Import modules\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib.pyplot as plt\nfrom os import walk\nimport imageio\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nimport gc #garbage collection\n\n\nsampling=8900\nheight=512\nwidth=512\nclasses = {0 : \"0\", 1 : \"1\", 2 : \"2\", 3 : \"3\", 4 : \"4\", 5 : \"5\"} #5 grades ISUP\nbatch_size = 0\n#Standard preprocessing \n#load csv\n#base_path='../input/explo-and-pre-processing'\n#train=pd.read_csv(f'{base_path}/train.csv') # /!\\ train is a bad name, it has training and validation in it. \n\n# just to be sure we will call every image contained in the folder\n#(_, _, list_of_filenames) = next(walk(f\"{base_path}/processed_pictures_rotation_included/\"))\n#list_of_filenames = [sub.replace('.tiff', '') for sub in list_of_filenames] \n#train=train[train['image_id'].isin(list_of_filenames)]\n\n# Here we need it to feed the tensorFlow \"tensor\" with the path to the images\n#train['path_to_image'] = train.apply (lambda x: f\"{base_path}/processed_pictures_rotation_included/{x['image_id']}.tiff\", axis=1)\n\n#final_validation=pd.read_csv(f'{base_path}/test.csv')\n\"\"\"\n#Tile 512 512\n#load csv\nbase_path='../input/panda-tile-512x512'\ntrain=pd.read_csv(f'../input/explo-and-pre-processing/train.csv') # /!\\ train is a bad name, it has training and validation in it. \nfinal_test=pd.read_csv(f'../input/prostate-cancer-grade-assessment/test.csv')\n\"\"\"\n#pre processing homemade \nbase_path='../input/tile-pre-processing/512x512x3/'\ntrain=pd.read_csv(f'../input/prostate-cancer-grade-assessment/train.csv') # /!\\ train is a bad name, it has training and validation in it. \nfinal_test=pd.read_csv(f'../input/prostate-cancer-grade-assessment/test.csv')\n\n# just to be sure we will call every image contained in the folder\n(_, _, list_of_filenames) = next(walk(f\"{base_path}/\"))\nlist_of_filenames = [sub.replace('.png', '') for sub in list_of_filenames] \ntrain=train[train['image_id'].isin(list_of_filenames)]\nif sampling:\n    train=train.sample(n=sampling)\nX_train, X_test, y_train, y_test = train_test_split(\n    train, train.isup_grade, test_size=0.33, random_state=42)\n\n\ndef decode_img(img):\n  # convert the compressed string to a 3D uint8 tensor\n  img = tf.image.decode_png(img, channels=3)\n  # resize the image to the desired size\n  return tf.image.resize(img, [height,width])\n\ndef process_path(file_path,label):\n    label = tf.one_hot(label, len(classes))\n    label = tf.reshape(label, (1,6))\n    # load the raw data from the file as a string\n    img = tf.io.read_file(file_path)\n    img = decode_img(img)\n    img = img/255\n    img=tf.reshape(img, [1,512,512,3])\n#     img = img.reshape(-1,512,512,3)\n    \n    return img, label\n\ndef get_label(file_path):\n    # convert the path to a list of path components\n    parts = tf.strings.split(file_path, os.path.sep)\n    # The second to last is the class-directory\n    one_hot = parts[-2] == class_names\n    # Integer encode the label\n    return tf.argmax(one_hot)\n    \nlist_ds = tf.data.Dataset.from_tensor_slices((base_path+X_train.image_id+'.png',X_train.isup_grade))\n#list_ds = tf.data.Dataset.list_files(base_path+'*', shuffle=False)\n\nfor f,i in list_ds.take(5):\n    print(f.numpy())\n    print(i.numpy())\ntrain_ds=list_ds\ntrain_ds = train_ds.map(process_path)\nfor image, label in train_ds.take(1):\n    print(\"Image : \", image)\n    print(\"Image shape: \", image.numpy().shape)\n    print(\"Label: \", label.numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" sauvegarde\n    X_train['np_image'] = X_train.apply (lambda x : (imageio.imread(f\"{base_path}{x['image_id']}.png\")/255).reshape([-1,512,512,3]), axis=1)\n    X_test['np_image'] = X_test.apply (lambda x : (imageio.imread(f\"{base_path}{x['image_id']}.png\")/255).reshape([-1,512,512,3]), axis=1)\n\n    X_train_np_image_from_df=np.stack(X_train['np_image'])\n    y_train=tf.one_hot(y_train, 6)\n    y_train=tf.reshape(y_train, [len(X_train),1,6])\n    del X_train\n    #X_train_isup_from_df=np.array(X_train['isup_grade'])\n    X_test_np_image_from_df=np.stack(X_test['np_image'])\n    y_test=tf.one_hot(y_test, 6)\n    y_test=tf.reshape(y_test, [len(X_test),1,6])\n    del X_test\n    gc.collect()\n    #X_test_isup_from_df=np.array(X_test['isup_grade'])\n\n\n\n\n\n\n\n\n    #useful for TF classes\n    \n    # Here we will feed the tensorFlow database \n\n    training_dataset = (\n        tf.data.Dataset.from_tensor_slices(\n            (\n                X_train_np_image_from_df,\n                y_train\n                )\n        )\n    )\n\n    del(X_train_np_image_from_df)\n    validation_dataset = (\n        tf.data.Dataset.from_tensor_slices(\n            (\n                X_test_np_image_from_df,\n                y_test\n                )\n        )\n    )\n    del(X_test_np_image_from_df)\n    gc.collect()\n    # garbage collection / deleting\n\n\n    batch_size = 0\n    width = 512\n    height= 512\n   \n    # Output preview\n    for features_tensor, target_tensor in training_dataset.take(5):\n        print(f'features:{features_tensor} target:{target_tensor}')\n        #The tensor has the image path, and the label coded.\n\n    width = 512\n    height= 512\n    # 0 means that it won't be batched\n    #the function takes the first and the second item of the tensorflow item\n    #thanks to map it will be passed \"row by row\"\n    def _parse_function(filename, label, h=height, w=width, rotating=True): \n        image = tf.io.read_file(filename) #reading the image in the memory\n        #image = tfio.experimental.image.decode_tiff(image, index=0, name=None) # tiff to numpy 1/2\n        image = tf.io.decode_png(image,dtype=tf.dtypes.uint8, name=None)\n        if rotating==True :\n            sh=tf.shape(tf.io.read_file(filename))\n            print(f\"{sh}\")\n            width, height = image.shape[0],image.shape[1]\n            print(f\"w={width}, h={height}\")\n            image = tf.image.convert_image_dtype(image, tf.float32) # converting it 2/2\n    #     \n        image = image/255 #normalisation\n        #image = tf.image.resize(image, [h, w]) #resize\n        image = tf.cast(image, tf.float32) # transforming into a tf.float object\n\n        return image, label\n\n    tr_dataset = training_dataset.map(_parse_function)\n    va_dataset = validation_dataset.map(_parse_function)\n    \n    if batch_size != 0:\n        training_dataset = training_dataset.batch(batch_size, drop_remainder=True) # drop_remainder is to drop the last batch which is not complete\n        validation_dataset = validation_dataset.batch(batch_size, drop_remainder=True)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\ndef sizeof_fmt(num, suffix='B'):\n    ''' by Fred Cirera,  https://stackoverflow.com/a/1094933/1870254, modified'''\n    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n        if abs(num) < 1024.0:\n            return \"%3.1f %s%s\" % (num, unit, suffix)\n        num /= 1024.0\n    return \"%.1f %s%s\" % (num, 'Yi', suffix)\n\nfor name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),\n                         key= lambda x: -x[1])[:10]:\n    print(\"{:>30}: {:>8}\".format(name, sizeof_fmt(size)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from keras.models import load_model\nfrom sklearn.datasets import load_files   \nfrom keras.utils import np_utils\nfrom glob import glob\nfrom keras import applications\nfrom keras.preprocessing.image import ImageDataGenerator \nfrom keras import optimizers\nfrom keras.models import Sequential,Model,load_model\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D,GlobalAveragePooling2D\nfrom keras.callbacks import TensorBoard,ReduceLROnPlateau,ModelCheckpoint\n\nbase_model = applications.resnet50.ResNet50(weights= None, include_top=False, input_shape= (height,width,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes= len(classes)\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\nx = Dropout(0.7)(x)\npredictions = Dense(num_classes, activation= 'softmax')(x)\nmodel = Model(inputs = base_model.input, outputs = predictions)\nfrom keras.optimizers import SGD, Adam\n# sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)\nadam = Adam(lr=0.0001)\nmodel.compile(optimizer= adam, loss='categorical_crossentropy', metrics=['accuracy',QuadraticWeightedKappa()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history=model.fit(train_ds, epochs = 55, batch_size = batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#history retrieving\nprint(history.history.keys())\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['Kappa'])\nplt.plot(history.history['val_Kappa'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#save model\nmodel.save('resnet50')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}