{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n#import numpy as np # linear algebra\n#import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n#import os\n#print(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Quick, Draw! Mini Project \n\nThe purpose of the miniproject is to classify the \"Quick Draw!\" dataset using a constructed CNN architecture. \n\nBelow an overview of the content of this notebook can be found."},{"metadata":{"_uuid":"62e0996d830341f58befb7e1ba4932761a2cd485"},"cell_type":"markdown","source":"\n### Overview\n\n- The Data\n- The Network\n- Breakdown of Results\n\n\n"},{"metadata":{"trusted":true,"_uuid":"d76f79a53ef534252cccf052497eafee7002e441","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#setup\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport ast\nimport json\nfrom PIL import Image, ImageDraw \npath_train = '../input/train_simplified/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91aeb7070ba293fe7b33dfec90c7d4934d4720f5"},"cell_type":"markdown","source":"## 1. The Data \nThe train data is stored in a zip file containing a csv file for each class. There is a total of 340 classes. \n### Load the Data\nIn the following small code snippet the csv file for the class \"cat\" is loaded. Here only the first 10 rows are loaded into a pandas DataFrame just to show the structure of the data. As can be seen each sample is described by 6 parameters; a country code, the drawing, a key id, whether it was recognised or not, a timestamp, and the label (a word)."},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true,"_uuid":"7ab651900fa7eb697213032a236c2959ad5949b5"},"cell_type":"code","source":"cat=pd.read_csv('../input/train_simplified/cat.csv',nrows=10)\ncat","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"577db109aa85ad27e0ba1c876f0d5a2740696d3f"},"cell_type":"markdown","source":"### a. How many images does the dataset consist of?"},{"metadata":{"trusted":true,"_uuid":"943f2df6541916665807e18af135e7929aced123","_kg_hide-input":true},"cell_type":"code","source":"#load \nclassSamples=[]\nfilenames=[]\nfor filename in os.listdir(path_train):\n    filenames.append(filename)\n    temp=pd.read_csv(os.path.join(path_train,filename),usecols=['word'])\n    classSamples.append(temp.shape[0])\nprint(\"The dataset consists of \",sum(classSamples),\" images.\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6c76245c566c5a6ceb591da74e21872d6526fdac"},"cell_type":"markdown","source":"### b. How many classes? How many images per class?"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"de4d64ac806b7a5a1cfe7af363a7cf83c4e7b766"},"cell_type":"code","source":"minSamples=min(classSamples)\nprint(\"There are \",len(filenames),\" classes, and there are between \",minSamples,\" and \",max(classSamples),\" samples per class.\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"45975433f196c9c44899c92dfed540a18d9dbeb2"},"cell_type":"markdown","source":"### c.  Show 20 sample images from at least 8 different classes\nBelow 20 images from 10 randomly selected classes are shown."},{"metadata":{"trusted":true,"_uuid":"913e1398c95c6c6a2f59ab57876a87885e1a3cbc","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"display_samples=pd.DataFrame()\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/cat.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/owl.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/dog.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/ant.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/alarm clock.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/key.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/trumpet.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/donut.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/feather.csv',usecols=['drawing', 'word'],nrows=2))\ndisplay_samples=display_samples.append(pd.read_csv('../input/train_simplified/frog.csv',usecols=['drawing', 'word'],nrows=2))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"479fac1a66f9a288bba344785ba76934705b5a54"},"cell_type":"code","source":"#display_samples['drawing'] = display_samples['drawing'].apply(ast.literal_eval)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"2e1bc81f0d331d4967e50e89d49c6886bbec55dc"},"cell_type":"code","source":"display_samples['drawing'] = display_samples['drawing'].apply(json.loads)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a77400ec8f63af02f318b6c2490be2f2d631c906","_kg_hide-input":true},"cell_type":"code","source":"figrows=4\nfigcols=5\nfig, axs = plt.subplots(nrows=figrows, ncols=figcols, sharex=True, sharey=True, figsize=(16, 10))\nfor i, drawing in enumerate(display_samples.drawing):\n    ax = axs[i // figcols, i % figcols]\n    for x, y in drawing:\n        ax.set_title(display_samples.word.iloc[i])\n        ax.plot(x, -np.array(y), lw=3)\n    ax.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7f3c417eacaf1a5e926c9a7e49d999b70643a8a"},"cell_type":"markdown","source":"### d. Consider if/how the data distribution will affect training of a classifier.\n\nIf the network training the model is presented with more samples from a certain class than other, the model is likely to be more specialised in the traits of the given class  and thus perform better when classifying that specific class. If there is a significant imbalance in the amount of samples from the different classes presented during training the training of recognition of some classes might overrule the training of other classes resulting in a poor perfomance when classifying the overruled classes."},{"metadata":{"_uuid":"d3325ca6e88ffcea61a917483446629f067f7311"},"cell_type":"markdown","source":"## 2. The Network\nThe intend is to create a Convolutional Neural Network which trains on the data in an image form rather than the \"stroke\" form in the provided dataset. \n\nIn the following code snippet the csv file for each class is loaded. From each file a pandas DataFrame is extracted and appended to a collective DataFrame, which will serve as the training data containing datapoints from all the classes.  The kernel dies if all data about all samples is loaded, therefore the amount of data kept in structures has ben minimised. In order minimise memory use only the columns \"Drawing\" and \"Word\", which is the label, and in some cases \"Recognized\" will be used. Furthermore, only 500 samples from each class is used. "},{"metadata":{"trusted":true,"_uuid":"622e435bea8f6342d328bba44cf9fbec580a2065"},"cell_type":"code","source":"%reset -f ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef1cad9c086e020aefec21ceb76e50e75185c7bd","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#setup\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport graphviz\n\nfrom tqdm import tqdm\nimport ast\nimport cv2\nfrom glob import glob\nfrom PIL import Image, ImageDraw \nfrom dask import bag\nfrom numpy import array\nfrom numpy import argmax\nfrom sklearn.utils import shuffle\nimport multiprocessing as mp\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers.normalization import BatchNormalization\npath_train = '../input/train_simplified/'\n\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\nfrom keras.utils import plot_model\n#import pydotplus as pydot\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bcae8f99fcb308897889d4d1fb295c529071f41c"},"cell_type":"markdown","source":"A function is defined for converting from strokes to an image"},{"metadata":{"trusted":true,"_uuid":"d6be78ab2957e0c15143bb00c32dfbf6c81016ba"},"cell_type":"code","source":"#Definitions of functions\n#imheight, imwidth = 32, 32 \nimheight, imwidth = 64, 64  \n# function for converting from strokes into image\ndef draw_it(strokes):\n    image = Image.new(\"P\", (256,256), color=255)\n    image_draw = ImageDraw.Draw(image)\n    for stroke in ast.literal_eval(strokes):\n        for i in range(len(stroke[0])-1):\n            image_draw.line([stroke[0][i], \n                             stroke[1][i],\n                             stroke[0][i+1], \n                             stroke[1][i+1]],\n                            fill=0, width=5)\n    image = image.resize((imheight, imwidth))\n    return np.array(image)/255.\nprint(\"Functions defined\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df3484c8c1d046f02a5506242ba309a064923f41"},"cell_type":"markdown","source":"Data is loaded and converted"},{"metadata":{"trusted":true,"_uuid":"acc6c7024cdb398057ddfbb4315fb51c5372aa66"},"cell_type":"code","source":"train_data=pd.DataFrame()\nsamples_per_class=500\nfor filename in os.listdir(path_train):\n    #filenames.append(filename)\n    #temp=pd.read_csv(os.path.join(path_train,filename),usecols=['drawing','word'],nrows=number_of_samples_per_class)\n    temp=pd.read_csv(os.path.join(path_train,filename),usecols=['countrycode','drawing', 'recognized','word'],nrows=samples_per_class*5//4)\n    temp=temp[temp.recognized == True].head(samples_per_class)\n    temp=temp.loc[:,['countrycode','drawing','word']]\n    train_data=train_data.append(temp)\nprint(\"dataload done\")\nfeatureVector = []\nn_cores = mp.cpu_count()\npool = mp.Pool(processes=n_cores//2)\nfeatureVector = pool.map_async(draw_it,train_data.drawing).get()\npool.close()\npool.join()\nprint(\"FINISHED parallel for train data\")\ntrain_data['drawing'] =featureVector\n\ndel featureVector\ndel temp","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"d6479835a62288e434c76baa3cb29c71d415fb61"},"cell_type":"code","source":"## for showing examples\n#print(train_data.word.iloc[3])\n#plt.imshow(train_data.drawing.iloc[3])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d148af3381af36cb1712c0d2a2c98d9ce91e2db"},"cell_type":"markdown","source":"Splitting in train and test sets and encoding labels and countries "},{"metadata":{"trusted":true,"_uuid":"6ef41fba03425641ce33494fb987c7e7eef00ed1"},"cell_type":"code","source":"#making sure the dataset is randomly mixed before splittiong\ntrain_data=shuffle(shuffle(train_data,random_state=33),random_state=27)\nlabel_encoder=LabelEncoder()\nonehot_encoder=OneHotEncoder(sparse=False)\ntrain_data.countrycode=train_data.countrycode.fillna('0')\nunique_countries=train_data.countrycode.unique()\nnum_countries=unique_countries.shape[0]\n\ncountry_encoder=LabelEncoder()\ncountry_encoder.fit(unique_countries)\n\n\ntemp_label=label_encoder.fit_transform(train_data.word)\nforOneHot = temp_label.reshape(len(temp_label), 1)\n#print(forOneHot)\nlabel=onehot_encoder.fit_transform(forOneHot)\n#print(label)\n#train_X,validation_X,train_y,validation_y = train_test_split(train_data,label,stratify = label,test_size=0.20)\ntrain_X,validation_X,train_y,validation_y=train_test_split(train_data, label, test_size=0.2, random_state=13)\n#train_x=np.stack(np.array(train_X.drawing))\n#train_x=train_X.drawing.tolist()\n#validation_image=validation_X.drawing\n\ndel forOneHot\ndel temp_label\ndel train_data\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c0d713c64cb12648eee0e2eb77a7dea717cc13a8"},"cell_type":"markdown","source":"converting split data to the right format"},{"metadata":{"trusted":true,"_uuid":"2c5cbc4807a9addeddbe2fe840bf9308a1407d64"},"cell_type":"code","source":"train_x=np.array(train_X.drawing.tolist())\nvalidation_x=np.array(validation_X.drawing.tolist())\ntrain_x = train_x.reshape(train_x.shape[0], imheight, imwidth, 1)\nvalidation_x = validation_x.reshape(validation_x.shape[0], imheight, imwidth, 1)\nvalidation_cc=country_encoder.transform(validation_X.countrycode)\ndel validation_X\ndel train_X","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8566e853465c6a3084d6d4ade66bac17c60a538"},"cell_type":"markdown","source":"Creating the structure of the network"},{"metadata":{"trusted":true,"_uuid":"c9a2464ac8bc560e59b8ee9ed67d083b1a0619d5"},"cell_type":"code","source":"batch_size = 150\nepochs = 50\ninput_shape = train_x[0].shape\n#input_shape = train_image.iloc[0].shape\nprint(input_shape)\nnum_classes = 340\n    \nmodel = Sequential()\nmodel.add(Conv2D(12, kernel_size=(3, 3),\n                     activation='relu',\n                     input_shape=input_shape,\n                     data_format='channels_last',\n                     padding = \"same\"))\n\nmodel.add(Conv2D(12, kernel_size=(3, 3),\n                     activation='relu',\n                     input_shape=input_shape,\n                     data_format='channels_last',\n                     padding = \"same\"))\n#model.add(MaxPooling2D(pool_size=(2, 2),data_format='channels_last'))\n    #model.add(Dropout(0.25))\nmodel.add(Conv2D(32, kernel_size=(5, 5),\n                     activation='relu',\n                     input_shape=input_shape,\n                     data_format='channels_last',\n                     padding = \"same\"))\n\nmodel.add(Conv2D(32, kernel_size=(5, 5),\n                     activation='relu',\n                     input_shape=input_shape,\n                     data_format='channels_last',\n                     padding = \"same\"))\nmodel.add(BatchNormalization())\nmodel.add(Conv2D(32,\n                     kernel_size=(5, 5),\n                     activation='relu',\n                     padding = \"same\",\n                     data_format='channels_last'))\nmodel.add(MaxPooling2D(pool_size=(2, 2),data_format='channels_last'))\n    \nmodel.add(Conv2D(64,\n                     kernel_size=(5, 5),\n                     activation='relu',\n                     padding = \"same\",\n                     data_format='channels_last'))\nmodel.add(MaxPooling2D(pool_size=(2, 2),data_format='channels_last'))\n    \nmodel.add(Conv2D(64,\n                     kernel_size=(5, 5),\n                     activation='relu',\n                     padding = \"same\",\n                     data_format='channels_last'))\n    \nmodel.add(Flatten())\nmodel.add(Dense(1024, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\nmodel.add(Dense(1024, activation='relu'))\nmodel.add(Dropout(0.25))\nmodel.add(Dense(num_classes, activation='softmax',))\n    \nlearningrate = 1e-2\nadagrad = keras.optimizers.Adagrad(lr=learningrate, epsilon=None, decay=0.0005)\n    #model.compile(loss='categorical_crossentropy', optimizer=sgd)\nmodel.compile(loss='categorical_crossentropy',\n                  optimizer=adagrad,\n                  metrics=['accuracy'])\nmodel.summary()\nprint(\"Structure made\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca116e621183d7c32cabd87aaf2b00a5f11bf096"},"cell_type":"markdown","source":"Training the model"},{"metadata":{"trusted":true,"_uuid":"9d6c45697087bfac4e735eaf4d3f3e838275c144"},"cell_type":"code","source":"history = model.fit(train_x, train_y,\n              batch_size=batch_size,\n              epochs=epochs,\n              shuffle=True,\n              verbose=1,\n              validation_split=0.1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"698c068124c446c9a45b161b8bba04452f91a7f5"},"cell_type":"markdown","source":"Evaluation of training."},{"metadata":{"_uuid":"3799d59bc82327f3eb1457c3a4876d4479cabbd8","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"\"\"\"\n#Could be used for plotting the CNN structure but only wors sometimes\nfrom keras.utils import plot_model\nplot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))\n\n\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb623fc5e18a2e33fe71ea5e3498f98999b2646f","_kg_hide-input":false},"cell_type":"code","source":"    print(history)\n    fig1, ax_acc = plt.subplots()\n    plt.plot(history.history['acc'])\n    plt.plot(history.history['val_acc'])\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.title('Model - Accuracy')\n    plt.legend(['Training', 'Validation'], loc='lower right')\n      \n    plt.show()\n    \n    fig2, ax_loss = plt.subplots()\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title('Model- Loss')\n    plt.legend(['Training', 'Validation'], loc='upper right')\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.legend(['Training', 'Validation'], loc='lower right')\n      \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b57edb917a0ecfef9c7676b041679618655a65c"},"cell_type":"markdown","source":"Classify test dataset"},{"metadata":{"trusted":true,"_uuid":"280db704db8f4049e70a4a87fcc1e61a23d5ec55"},"cell_type":"code","source":"score = model.evaluate(validation_x, validation_y, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1]*100)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e6102b38e0aaa5e9cc991469b18b76c4678c507"},"cell_type":"markdown","source":"Examination of accuracies"},{"metadata":{"trusted":true,"_uuid":"56c6064124772893305dcfc6020b28d60f43391e","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"Pred_label=model.predict(validation_x)\nPredict_label=Pred_label.dot(onehot_encoder.active_features_).astype(int)\nNum_real_y=validation_y.dot(onehot_encoder.active_features_).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e8b5168c5e2d0fb79b2c72957a78bc4d3806cfa"},"cell_type":"code","source":"Class_perform=np.zeros([num_classes,2])\nClass_Acc=np.zeros([num_classes])\nfor i in range(Num_real_y.shape[0]):\n    if Predict_label[i]==Num_real_y[i]:\n        Class_perform[Num_real_y[i],0]+=1\n    Class_perform[Num_real_y[i],1]+=1\n    for k in range(num_classes):\n        if Class_perform[k,1]>0:\n            Class_Acc[k]=Class_perform[k,0]/Class_perform[k,1]\nindices=Class_Acc.argsort()[::-1][:5]\nprint(\"Top 5 classified classes: \",label_encoder.inverse_transform(indices))\nprint(\"Accuracy of top 5 classified classes: \",Class_Acc[indices])\nprint(\"Sample amount of top 5 classified classes: \",Class_perform[indices,1])\nprint(\"Worst Class :\",label_encoder.inverse_transform(Class_Acc.argsort()[0]))\nprint(\"Accuracy of worst class :\",Class_Acc[Class_Acc.argsort()[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"832319e63049a88dec336578ab5a2da41c0e87af"},"cell_type":"code","source":"Country_perform=np.zeros([num_countries,2])\nCountry_Acc=np.zeros([num_countries])\nfor i in range(Num_real_y.shape[0]):\n    if Predict_label[i]==Num_real_y[i]:\n        Country_perform[validation_cc[i],0]+=1\n    Country_perform[validation_cc[i],1]+=1\n    for k in range(num_countries):\n        if Country_perform[k,1]>0:\n            Country_Acc[k]=Country_perform[k,0]/Country_perform[k,1]\nindicesC=Country_Acc.argsort()[::-1][:5]\nprint(\"Top 5 classified countries: \",country_encoder.inverse_transform(indicesC))\nprint (\"Accuracy of top 5 classified countries: \",Country_Acc[indicesC])\nprint(\"Sample amount from top 5 classified countries: \",Country_perform[indicesC,1])\nprint(\"Worst country :\",country_encoder.inverse_transform(Country_Acc.argsort()[1]))\nprint(\"Accuracy of worst country :\",Country_Acc[Country_Acc.argsort()[1]])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa39238e31dbf9b663cf93b02ab3c75bd28ab7e2"},"cell_type":"markdown","source":"The results can vary from run to run with the same settings. This might be due to the fact that there seemingly is no random seed used for when the train data is shuffled inbetween epochs. "},{"metadata":{"_uuid":"f4cb6f88f400e77e1a5a4cb68164a8411da3d2d7"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}