{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Imports of packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nimport ast\nimport json\n\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom keras.utils import to_categorical\nfrom PIL import Image, ImageDraw\n\nfrom keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D ,Dropout, Flatten\nfrom keras.layers import Dense\nimport tensorflow as tf\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\n\npath = '/kaggle/input/quickdraw-doodle-recognition/train_simplified/' # use your path\nall_files = glob.glob(path + \"/*.csv\")\nnb_files=len(all_files)\nnb_mots=34\nn_sample_perfile=10000\nnb_lignes_sample=340000\ntest_size_val=0.33\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load of the files\n\nI decided to look only to a few classes in this notebook. The dataset contains a total of 340 words. \n\nThe first step to load the files is to read the .csv file and load them in a DataFrame"},{"metadata":{"trusted":true},"cell_type":"code","source":"li = []\ncount=0\n\nfor filename in all_files[0:nb_mots]:\n    print(filename)\n    df_reduced = pd.read_csv(filename, index_col=None, header=0, nrows=50000)\n    df_reduced=df_reduced[df_reduced['recognized']==True].drop(['timestamp','recognized'], axis=1).sample(n=n_sample_perfile, random_state=24)\n    \n    count+=1\n    print(df_reduced.size,count ,'/',nb_files)\n    li.append(df_reduced)\n\nprint('load done for',nb_mots,'files')\nframe = pd.concat(li, axis=0, ignore_index=True)\nframe = frame.sample(n=n_sample_perfile*nb_mots, random_state=24)\n\n\ndel li\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(frame.shape)\nframe.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" #  Data preprocessing:\n \nIn this section, I replace the words containing spaces by underscores.\nI convert then the category into a one hot.\n\nI tried first to use ast.litteral_eval. I changed to json.load which is much faster"},{"metadata":{"trusted":true},"cell_type":"code","source":"#replace spaces in words by _\nframe['word'] = frame['word'].str.replace(' ','_')\n\n#transform drawing into arrays (can take some time)\nframe['drawing']=frame['drawing'].apply(json.loads)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categories=frame['word'].unique()\nprint(categories)\nlabel_encoder = LabelEncoder()\ninteger_encoded = label_encoder.fit_transform(categories)\none_hot_encoded= to_categorical(integer_encoded,dtype='float32')\ndict_words_one_hot = dict(zip(categories, one_hot_encoded))\n#map one hot incoding in the DataFrame\nframe['word_encoded']=frame.word.map(dict_words_one_hot)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"frame.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Overview of the data\n\nThis section is used to see an overview of the drawings, I added the name of each drawing as the legend of the plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"N=10\noverview_sample=frame.sample(n=100, random_state=24).reset_index()\n\n\nfig1, axes = plt.subplots(N,N ,sharex=True, sharey=True, figsize=(20, 15))\nfor index, rows in enumerate(overview_sample['drawing']):\n    word=overview_sample.iloc[index,4]\n    ax1 = axes[index // N, index % N]\n    \n    for x,y in rows:\n        \n        # -np.array is used to inverse the y-axis here\n        ax1.plot(x,-np.array(y),marker='o',markersize=3,linewidth=2)\n    ax1.axis('off')\n    ax1.set(title=str(index)+' - '+ word)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Conversions fonctions\n\nUsed to convert the drawing into a img in a numpy array \n\nfunctions from: https://www.kaggle.com/gaborfodor/how-to-draw-an-owl-lb-0-002"},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_to_np_raw(drawing, width = 256, height = 256):\n    \"\"\"\n    INPUT:\n        drawing - drawing in initial format\n        width - width of the initial image\n        height - height of the initial image\n    OUTPUT:\n        img - drawing converted to the numpy array (28 X 28)\n    \"\"\"\n    # initialize empty numpy array\n    img = np.zeros((28, 28))\n    \n    # create a PIL image out of drawing\n    pil_img = convert_to_PIL(drawing)\n    \n    #resize to 28,28\n    pil_img.thumbnail((28,28), Image.ANTIALIAS)\n    \n    pil_img = pil_img.convert('RGB')\n    pixels = pil_img.load()\n    \n    # fill in numpy array with pixel values\n    for i in range(0, 28):\n        for j in range(0, 28):\n            img[i, j] = 1 - pixels[j, i][0] / 255\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_to_PIL(drawing, width = 256, height = 256):\n    \"\"\"\n    Function to convert from drawing to .\n    INPUT:\n        drawing - drawing from 'drawing' column\n        width - width of the initial image\n        height - height of the initial image\n    OUTPUT:\n        pil_img - (PIL Image) image\n    \"\"\"\n    \n    # initialize empty (white) PIL image\n    pil_img = Image.new('RGB', (width, height), 'white')\n    pixels = pil_img.load()\n            \n    draw = ImageDraw.Draw(pil_img)\n    \n    # draw strokes as lines\n    for x,y in drawing:\n        for i in range(1, len(x)):\n            draw.line((x[i-1], y[i-1], x[i], y[i]), fill=0)\n        \n    return pil_img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" Then we need to convert the pictures into an np.array to be able to predict them.\n This is currently a bottleneck in the analysis, because the transformation takes a lot of time. The RAM usage is limitating the number of sample used. \n \n Solution could be to try to parallelize this line."},{"metadata":{"trusted":true},"cell_type":"code","source":"frame['drawing']=frame['drawing'].apply(convert_to_np_raw)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Then we create the Train and Test Datasets\n\nI decided to drop all not useful columns, however it could be interessant to consider the countrycode as a feature. Some countries can have similar ways to draw doodles based on their location."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(frame.drop(['countrycode','word','word_encoded','key_id'],axis=1), frame['word_encoded'], test_size=test_size_val, random_state=24)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction\n\nOne of the current limitation in this analysis is the number of samples and the number of words."},{"metadata":{},"cell_type":"markdown","source":"# Model 1 - Very basic model but efficient"},{"metadata":{},"cell_type":"markdown","source":"For this first model, I decided to use a very basic Neural Network using Dense layer with a 'relu' activation. And a softmax activation for the output layer. "},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(10, activation='relu', input_shape=(784,)))\nmodel.add(Dense(10, activation='relu'))\nmodel.add(Dense(nb_mots,activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I used adam as an "},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='adam', \n           loss='categorical_crossentropy', \n           metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=X_train['drawing'].apply(lambda v: v.reshape(784))\ntest_data=X_test['drawing'].apply(lambda v: v.reshape(784))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We convert the DF to numpy array\nX_train_stacked=np.stack(train_data.values)\ny_train_stacked=np.stack(y_train.values)\nX_test_stacked=np.stack(test_data.values)\ny_test_stacked=np.stack(y_test.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I then had to convert the y_train and y_test to numpy array with the right shape so they can be converted by Keras to Tensors"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train=np.concatenate( y_train.to_numpy(), axis=0 ).reshape(y_train.shape[0],nb_mots)\ny_test=np.concatenate( y_test.to_numpy(), axis=0 ).reshape(y_test.shape[0],nb_mots)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(np.stack(train_data.values),y_train, validation_split=0.2, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(np.stack(test_data.values), y_test_stacked)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"> # Model 2 - Conv approach"},{"metadata":{},"cell_type":"markdown","source":"### First we need to use the right shape for the values for the convolutional network"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Here the images are reshaped to (28,28,1) array\nX_train_new=X_train['drawing'].apply(lambda v: v.reshape(28,28,1))\nX_test_new=X_test['drawing'].apply(lambda v: v.reshape(28,28,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = Sequential()\nmodel2.add(Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(28, 28, 1)))\nmodel2.add(MaxPooling2D(pool_size=(2, 2)))\nmodel2.add(Conv2D(64, kernel_size=(3, 3), padding='same', activation='relu'))\nmodel2.add(MaxPooling2D(pool_size=(2, 2)))\nmodel2.add(Dropout(0.2))\nmodel2.add(Flatten())\nmodel2.add(Dense(680, activation='relu'))\nmodel2.add(Dropout(0.5))\nmodel2.add(Dense(nb_mots, activation='softmax'))\nmodel2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.compile(optimizer='adam', \n           loss='categorical_crossentropy', \n           metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.fit(np.stack(X_train_new.values),y_train, validation_split=0.2, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.evaluate(np.stack(X_test_new.values),y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 3 - More classic but efficient "},{"metadata":{"trusted":true},"cell_type":"code","source":"model3 = Sequential()\nmodel3.add(Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(28, 28, 1)))\nmodel3.add(Conv2D(16, 3, padding='same', activation='relu'))\nmodel3.add(MaxPooling2D())\nmodel3.add(Conv2D(32, 3, padding='same', activation='relu'))\nmodel3.add(MaxPooling2D())\nmodel3.add(Conv2D(64, 3, padding='same', activation='relu'))\nmodel3.add(MaxPooling2D())\nmodel3.add(Flatten())\nmodel3.add(Dense(128, activation='relu'))\nmodel3.add(Dense(nb_mots, activation='softmax'))\nmodel3.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model3.compile(optimizer='adam', \n           loss='categorical_crossentropy', \n           metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model3.fit(np.stack(X_train_new.values),y_train, validation_split=0.2, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model3.evaluate(np.stack(X_test_new.values),y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 4 - Adding dropout to model 3"},{"metadata":{"trusted":true},"cell_type":"code","source":"model4 = Sequential()\nmodel4.add(Conv2D(32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(28, 28, 1)))\nmodel4.add(Conv2D(16, 3, padding='same', activation='relu'))\nmodel4.add(MaxPooling2D())\nmodel4.add(Conv2D(32, 3, padding='same', activation='relu'))\nmodel4.add(MaxPooling2D())\nmodel4.add(Conv2D(64, 3, padding='same', activation='relu'))\nmodel4.add(MaxPooling2D())\nmodel4.add(Flatten())\nmodel4.add(Dense(128, activation='relu'))\nmodel4.add(Dropout(0.4))\nmodel4.add(Dense(nb_mots, activation='softmax'))\nmodel4.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model4.compile(optimizer='adam', \n           loss='categorical_crossentropy', \n           metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model4.fit(np.stack(X_train_new.values),y_train, validation_split=0.2, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model4.evaluate(np.stack(X_test_new.values),y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Summary"},{"metadata":{},"cell_type":"markdown","source":"# Next steps to go further:\n\nSeveral steps that I still want to try to implement: \n- Try to fit the model using more sample: BatchApproach\n- Try to parallelize bottleneck steps and apply functions\n- Optimization of layers and parameters and number of epoch\n- Use pre-models such as MobileNet to check their scores\n- Try to use LTSM layers in addition to Conv/Pooling layers"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}