{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Quick,Draw Doodle\n\n\n* The framework used Keras.\n\n* Data is to learn a total of 340 classes to fit the description of the picture. Since you only need to work with the kernel, it may be more advantageous to use simplified data.\n\n* For most of these projects, mobilenet has the best performance. but, I proceeded using resnet50 due to constraints such as time."},{"metadata":{},"cell_type":"markdown","source":"In this section you will:\n\n* Receive data.\n* After confirming this with pandas, check with the picture.\n* We will run the preprocessing to utilize the model in the future.\n* After preprocessing, we divide it into train and validation.\n\nFor the code, see https://www.kaggle.com/keunyoungjung/kynet-quickdraw."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom PIL import Image , ImageDraw\nfrom sklearn.preprocessing import *\nimport time\nimport ast\nimport os\nimport tensorflow as tf\nfrom keras import models, layers\nfrom keras import Input\nfrom keras.models import Model, load_model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import optimizers, initializers, regularizers, metrics\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom keras.layers import BatchNormalization, Conv2D, Activation , AveragePooling2D\nfrom keras.layers import Dense, GlobalAveragePooling2D, MaxPooling2D, ZeroPadding2D, Add\nfrom keras.models import Sequential\nfrom keras.metrics import top_k_categorical_accuracy\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom tqdm import tqdm\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        fpath = os.path.join(dirname, filename)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(dirname+'/'+'cat.csv')\ndf['word'] = df['word'].replace(' ','_',regex = True)\nprint(type(df['recognized'][0]))\n\nidx= df.iloc[:5].index\nprint(df.loc[idx,'recognized'].values)\n\nfor i in range(len(df.loc[idx,'drawing'].values)) :\n    if df.loc[idx,'recognized'].values[i] == True :\n        print(i, end=' ')\n\nidx= df.iloc[:2000].index\nT_cnt = 0\nF_cnt = 0\nfor i in range(len(df.loc[idx,'drawing'].values)) :\n    if df.loc[idx,'recognized'].values[i] == True :\n        T_cnt += 1\n    else : F_cnt += 1\n\nprint('\\nTrue Count :',T_cnt)\nprint('False Count :',F_cnt)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_draw(img_arr) :\n    k=3\n    for i in range(len(img_arr[k])):\n        img = plt.plot(img_arr[k][i][0],img_arr[k][i][1])\n        plt.scatter(img_arr[k][i][0],img_arr[k][i][1])\n    plt.xlim(0,256)\n    plt.ylim(0,256)\n    plt.gca().invert_yaxis()\n\nten_ids = df.iloc[:10].index\nimg_arr = [ast.literal_eval(lst) for lst in df.loc[ten_ids,'drawing'].values]  #ast.literal_eval is squence data made string to array\nprint(img_arr[3])\ncheck_draw(img_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_img(img_arr) :\n    image = Image.new(\"P\", (256,256), color=255)\n    image_draw = ImageDraw.Draw(image)\n    for stroke in img_arr:\n        for i in range(len(stroke[0])-1):\n            image_draw.line([stroke[0][i], \n                             stroke[1][i],\n                             stroke[0][i+1], \n                             stroke[1][i+1]],\n                            fill=0, width=5)\n    return image\nimg = make_img(img_arr[3])\nimg = img.resize((64,64))\nplt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bar = '□□□□□□□□□□'\nsw = 1\ndef percent_bar(array,count,st_time):   #퍼센트를 표시해주는 함수\n    global bar\n    global sw\n    length = len(array)\n    percent = (count/length)*100\n    spend_time = time.time()-st_time\n    if count == 1 :\n        print('preprocessing...')\n    print('\\r'+bar+'%3s'%str(int(percent))+'% '+str(count)+'/'+str(length),'%.2f'%(spend_time)+'sec',end='')\n    if sw == 1 :\n        if int(percent) % 10 == 0 :\n            bar = bar.replace('□','■',1)\n            sw = 0\n    elif sw == 0 :\n        if int(percent) % 10 != 0 :\n            sw = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing(filenames) :\n    img_batch = 2000\n    X= []\n    Y= []\n    class_label = []\n    st_time = time.time()\n    class_num = 340\n    Y_num = 0\n    for fname in filenames[0:class_num] :\n        percent_bar(filenames[0:class_num],Y_num+1,st_time)\n        df = pd.read_csv(os.path.join(dirname,fname))\n        df['word'] = df['word'].replace(' ','_',regex = True)\n        class_label.append(df['word'][0])\n        keys = df.iloc[:img_batch].index\n        #print(len(keys))\n        \n        for i in range(len(df.loc[keys,'drawing'].values)) :\n            if df.loc[keys,'recognized'].values[i] == True :\n                drawing = ast.literal_eval(df.loc[keys,'drawing'].values[i])\n                img = make_img(drawing)\n                img = np.array(img.resize((64,64)))\n                img = img.reshape(64,64,1)\n                X.append(img)\n                Y.append(Y_num)\n        Y_num += 1\n        \n    tmpx = np.array(X)\n\n    Y = np.array([[i] for i in Y])\n    enc = OneHotEncoder(categories='auto')\n    enc.fit(Y)\n    tmpy = enc.transform(Y).toarray()\n    \n    del X\n    del Y     #RAM메모리 절약을 위해 사용하지 않는 변수 삭제\n    \n    return tmpx , tmpy , class_label , class_num\n\ntmpx , tmpy , class_label , class_num = preprocessing(filenames)\nprint('\\n',tmpx.shape, tmpy.shape, '\\n5th class : ',class_label[0:5])\n#df.head()\n#print(drawing[0])\n#img = make_img(drawing[1])\n#plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(tmpx,tmpy, test_size = 0.1,random_state = 0)\ndel tmpx\ndel tmpy     #RAM메모리 절약을 위해 사용하지 않는 변수 삭제\n\nprint(X_train.shape,X_val.shape,Y_train.shape,Y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# RESNET50\n\n* This code was written with reference to https://eremo2002.tistory.com/76.\n"},{"metadata":{},"cell_type":"markdown","source":"In this session, we will implement resnet50.\n\n* For conceptual information on resnet50, it is good to see: https://datascienceschool.net/view-notebook/958022040c544257aa7ba88643d6c032/\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"K = class_num\n \n \ninput_tensor = Input(shape=(64, 64, 1), dtype='float32', name='input')\n \n \ndef conv1_layer(x):    \n    x = ZeroPadding2D(padding=(3, 3))(x)\n    x = Conv2D(64, (7, 7), strides=(2, 2))(x)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = ZeroPadding2D(padding=(1,1))(x)\n \n    return x   \n \n    \n \ndef conv2_layer(x):         \n    x = MaxPooling2D((3, 3), 2)(x)     \n \n    shortcut = x\n \n    for i in range(3):\n        if (i == 0):\n            x = Conv2D(64, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n            \n            x = Conv2D(64, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n \n            x = Conv2D(256, (1, 1), strides=(1, 1), padding='valid')(x)\n            shortcut = Conv2D(256, (1, 1), strides=(1, 1), padding='valid')(shortcut)            \n            x = BatchNormalization()(x)\n            shortcut = BatchNormalization()(shortcut)\n \n            x = Add()([x, shortcut])\n            x = Activation('relu')(x)\n            \n            shortcut = x\n \n        else:\n            x = Conv2D(64, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n            \n            x = Conv2D(64, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n \n            x = Conv2D(256, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)            \n \n            x = Add()([x, shortcut])   \n            x = Activation('relu')(x)  \n \n            shortcut = x        \n    \n    return x\n \n \n \ndef conv3_layer(x):        \n    shortcut = x    \n    \n    for i in range(4):     \n        if(i == 0):            \n            x = Conv2D(128, (1, 1), strides=(2, 2), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)        \n            \n            x = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)  \n \n            x = Conv2D(512, (1, 1), strides=(1, 1), padding='valid')(x)\n            shortcut = Conv2D(512, (1, 1), strides=(2, 2), padding='valid')(shortcut)\n            x = BatchNormalization()(x)\n            shortcut = BatchNormalization()(shortcut)            \n \n            x = Add()([x, shortcut])    \n            x = Activation('relu')(x)    \n \n            shortcut = x              \n        \n        else:\n            x = Conv2D(128, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n            \n            x = Conv2D(128, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n \n            x = Conv2D(512, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)            \n \n            x = Add()([x, shortcut])     \n            x = Activation('relu')(x)\n \n            shortcut = x      \n            \n    return x\n \n \n \ndef conv4_layer(x):\n    shortcut = x        \n  \n    for i in range(6):     \n        if(i == 0):            \n            x = Conv2D(256, (1, 1), strides=(2, 2), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)        \n            \n            x = Conv2D(256, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)  \n \n            x = Conv2D(1024, (1, 1), strides=(1, 1), padding='valid')(x)\n            shortcut = Conv2D(1024, (1, 1), strides=(2, 2), padding='valid')(shortcut)\n            x = BatchNormalization()(x)\n            shortcut = BatchNormalization()(shortcut)\n \n            x = Add()([x, shortcut]) \n            x = Activation('relu')(x)\n \n            shortcut = x               \n        \n        else:\n            x = Conv2D(256, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n            \n            x = Conv2D(256, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n \n            x = Conv2D(1024, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)            \n \n            x = Add()([x, shortcut])    \n            x = Activation('relu')(x)\n \n            shortcut = x      \n \n    return x\n \n \n \ndef conv5_layer(x):\n    shortcut = x    \n  \n    for i in range(3):     \n        if(i == 0):            \n            x = Conv2D(512, (1, 1), strides=(2, 2), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)        \n            \n            x = Conv2D(512, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)  \n \n            x = Conv2D(2048, (1, 1), strides=(1, 1), padding='valid')(x)\n            shortcut = Conv2D(2048, (1, 1), strides=(2, 2), padding='valid')(shortcut)\n            x = BatchNormalization()(x)\n            shortcut = BatchNormalization()(shortcut)            \n \n            x = Add()([x, shortcut])  \n            x = Activation('relu')(x)      \n \n            shortcut = x               \n        \n        else:\n            x = Conv2D(512, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n            \n            x = Conv2D(512, (3, 3), strides=(1, 1), padding='same')(x)\n            x = BatchNormalization()(x)\n            x = Activation('relu')(x)\n \n            x = Conv2D(2048, (1, 1), strides=(1, 1), padding='valid')(x)\n            x = BatchNormalization()(x)           \n            \n            x = Add()([x, shortcut]) \n            x = Activation('relu')(x)       \n \n            shortcut = x                  \n \n    return x\n \n \n \nx = conv1_layer(input_tensor)\nx = conv2_layer(x)\nx = conv3_layer(x)\nx = conv4_layer(x)\nx = conv5_layer(x)\n \nx = GlobalAveragePooling2D()(x)\noutput_tensor = Dense(K, activation='softmax')(x)\n \nresnet50 = Model(input_tensor, output_tensor)\nresnet50.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# How does it work?\n\nIn this session, \n* we will check that the model is learning well in the same way as the graph.\n* It also performs preprocessing on test data.\n* Afterwards, create a file that matches the submission form."},{"metadata":{"trusted":true},"cell_type":"code","source":"def top_3_accuracy(x,y): \n    t3 = top_k_categorical_accuracy(x,y, 3)\n    return t3\n\nlearning_rate = 0.0001\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, \n                                   verbose=0, mode='auto', min_delta=0.005, cooldown=5, min_lr=learning_rate)\nearlystop = EarlyStopping(monitor='val_top_3_accuracy', mode='max', patience=3) \ncallbacks = [reduceLROnPlat, earlystop]\n\nresnet50.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', top_3_accuracy])\n\nhistory = resnet50.fit(x=X_train, y=Y_train,\n          batch_size = 128,\n          epochs = 3,\n          validation_data = (X_val, Y_val),\n          callbacks = callbacks,\n          verbose = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1,len(acc) + 1 )\n\nplt.plot(epochs, acc, 'bo' , label = 'Training Accuracy')\nplt.plot(epochs, val_acc, 'b' , label = 'Validation Accuracy')\nplt.title('Training and Validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs, loss, 'bo' , label = 'Training Loss')\nplt.plot(epochs, val_loss, 'b' , label = 'Validation Loss')\nplt.title('Training and Validation Loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing_test(df) :\n    X= []\n    keys = df.iloc[:].index\n    for i in tqdm(range(len(df.loc[keys,'drawing'].values))) :\n        drawing = ast.literal_eval(df.loc[keys,'drawing'].values[i])\n        img = make_img(drawing)\n        img = np.array(img.resize((64,64)))\n        img = img.reshape(64,64,1)\n        X.append(img)\n    \n    tmpx = np.array(X)\n    return tmpx\n\ntest = pd.read_csv(os.path.join('/kaggle/input/quickdraw-doodle-recognition', 'test_simplified.csv'))\nx_test = preprocessing_test(test)\nprint(test.shape, x_test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = x_test\npred = resnet50.predict(imgs, verbose=1)\ntop_3 = np.argsort(-pred)[:, 0:3]\n\n#print(pred)\nprint(top_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_pred = ['%s %s %s' % (class_label[k[0]], class_label[k[1]], class_label[k[2]]) for k in top_3]\nprint(top_3_pred[0:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df = pd.read_csv('/kaggle/input/quickdraw-doodle-recognition/sample_submission.csv', index_col=['key_id'])\npreds_df['word'] = top_3_pred\npreds_df.to_csv('subcnn_small.csv')\npreds_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}