{"cells":[{"metadata":{},"cell_type":"markdown","source":"캐글 과제 설명\n**제출 파일\n테스트 세트의 각 key_id에 대해 최대 3개의 워드 값을 예측해야 한다. 파일에는 헤더가 포함되어야 하며 다음과 같은 형식이 있어야 한다 중요: 어떤 \"단어\"는 한 단어 이상 교육 데이터는 이전에 출시된 Quick Draw 데이터 세트와 정렬되며, 공백으로 다중 단어 레이블을 구분한다. 이 경기의 카글 메트릭에는 공백이 없는 라벨이 필요하므로 공간을 밑줄로 대체하기 위해 라벨 예측을 조정해야 한다. 예를 들어, \"롤러 코스터\"는 \"롤러_코스터\"로 예측해야 한다.**"},{"metadata":{},"cell_type":"markdown","source":"기존 Quick, Draw! 데이터 세트에 대한 더 나은 분류자를 만드는 것이 과제이다. Kaggler는 이 데이터 세트에서 모델을 발전시킴으로써 패턴 인식 솔루션을 보다 광범위하게 개선할 수 있다. "},{"metadata":{},"cell_type":"markdown","source":"**import 필요 모듈**\n데이터 불러오기 및 확인"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport ast\nimport os\nfrom glob import glob\nfrom tqdm import tqdm\nfrom dask import bag\nfrom PIL import Image, ImageDraw \nimport cv2\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, DepthwiseConv2D, BatchNormalization, ZeroPadding2D\nfrom keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, AveragePooling2D \nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfrom keras.metrics import top_k_categorical_accuracy\nimport os\nfrom tqdm import tqdm #for문 처리과정 확인 함수 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rd=np.random.randint(340)#340까지의 수 중에 난수생성\npath = os.listdir('/kaggle/input/quickdraw-doodle-recognition/train_simplified') \n# df = pd.read_csv('/kaggle/input/quickdraw-doodle-recognition/train_simplified/'+path[rd])\n# print (df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"open_cv를 이용 이미지 그려주기\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"height = 64\nwidth = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_to_img(strokes):\n    BASE_SIZE = 256\n    image = Image.new(\"P\", (BASE_SIZE,BASE_SIZE), color=255) \n    image_draw = ImageDraw.Draw(image)\n    for each in ast.literal_eval(strokes):\n        for i in range(len(each[0])-1):\n            p1 = each[0][i],each[1][i]\n            p2 = each[0][i+1],each[1][i+1]\n            image_draw.line([p1,p2],width=5)\n    img = image.resize((height,width))\n    return np.array(img)/255.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"word에 띄어쓰기  '_' 로변경하기 \ntmp설정후 이미지확인"},{"metadata":{"trusted":true},"cell_type":"code","source":"# tmp = df['drawing'][20]\n# tmp2 = df['word'][20]\n# df['word']= df['word'].replace(' ','_',regex = True)\n# img = np.array(draw_to_img(tmp))\n# #데이터 형태확인\n# print(tmp)\n# print(' ')\n# print(tmp2)\n# plt.imshow(img)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test\nran=np.random.randint(340)#340까지의 수 중에 난수생성\nnums2names={i : v[:-4].replace(' ','_') for i , v in enumerate(path)}#불러온 데이터에 index 번호붙이기\nranclass=nums2names[ran]# 340까지의 수중에 랜덤으로 불러오기  \nranclass=ranclass.replace('_',' ')# 분류 제목을 _을 ' '로 바꿔주기\nrdpath='/kaggle/input/quickdraw-doodle-recognition/train_simplified/'+ranclass+'.csv' #랜덤으로 하나의 클래스 경로설정\none=pd.read_csv(rdpath,usecols=['drawing','recognized','word'],nrows=10) #10개 행의 drawing recognized word 불러오기\none=one[one.recognized==True].head(2)#그림 중 true인것 2개만불러오기\nname=one['word'].head(1)#one의 첫번째 word \nstrk=one['drawing']# one 의 drawing #2개\nname=name.values\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"데이터 추가 하기 "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_grand=[]\nnum_class = 340\nper_class = 800","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_paths = glob('/kaggle/input/quickdraw-doodle-recognition/train_simplified/*.csv')\nfor i , c in enumerate(tqdm(class_paths[0:num_class])): \n    train=pd.read_csv(c,usecols=['drawing','recognized'],nrows=per_class*1.5)\n    train=train[train.recognized==True].head(per_class)\n    imagebag=bag.from_sequence(train.drawing.values).map(draw_to_img)\n    train_array=np.array(imagebag.compute())#unmpy 형식\n    train_array=np.reshape(train_array,(per_class,-1))  #2000  -1로 reshpae로 행렬 형식 변경 그림형식 \n    label_array=np.full((train.shape[0],1),i)# label 붙여주기 , train.shape[0] = 2000\n    train_array=np.concatenate((label_array,train_array),axis=1)\n    train_grand.append(train_array)\n    del train_array\n    del label_array\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_grand=np.array([train_grand.pop() for i in np.arange(num_class)]) #데이터 세팅","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_grand=train_grand.reshape((-1,(height*width+1))) #64*64로 배열 변경\nprint(train_grand)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"setting to ( training set :  test set ) =  ( 9 : 1  )"},{"metadata":{"trusted":true},"cell_type":"code","source":"specific = 0.1 \nsequence_length = 50\ncut = int(specific * train_grand.shape[0])\nprint(cut)\n\nnp.random.shuffle(train_grand)\ny_train, X_train = train_grand[cut: , 0], train_grand[cut: , 1:]\ny_val, X_val = train_grand[0:cut, 0], train_grand[0:cut, 1:]\n\n# del train_grand\n\nx_train=X_train.reshape(X_train.shape[0],height,width,1)\nx_val=X_val.reshape(X_val.shape[0],height,width,1)\n\n\ny_train2 = keras.utils.to_categorical(y_train, num_class)\n\n\nprint(y_train.shape, \"\\n\",\n      x_train.shape, \"\\n\",\n      y_val.shape, \"\\n\",\n      x_val.shape)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"leaky_relu = tf.nn.leaky_relu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(ZeroPadding2D(padding=(1, 1),input_shape=(height,width,1)))\nmodel.add(Conv2D(32,kernel_size=(3,3),padding='same',activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2))) \nmodel.add(Conv2D(64,kernel_size=(3,3),padding='valid',activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(ZeroPadding2D(padding=(1, 1)))\nmodel.add(Conv2D(64,kernel_size=(3,3),padding='valid',activation=leaky_relu))\nmodel.add(AveragePooling2D(pool_size=(2,2)))\nmodel.add(ZeroPadding2D(padding=(1, 1)))\nmodel.add(Activation('relu'))\nmodel.add(DepthwiseConv2D(64, padding='same',activation='relu' ))\nmodel.add(AveragePooling2D(pool_size=(2,2)))\nmodel.add(BatchNormalization())\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(128,kernel_size=(3,3),padding='same',activation='relu'))\nmodel.add(Flatten())\nmodel.add(Dense(num_class*5, activation='relu'))\nmodel.add(Dense(num_class,activation='softmax'))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def top_3_accuracy(x, y):\n    return top_k_categorical_accuracy(x, y, k=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduceLROnPlat=ReduceLROnPlateau(monitor='val_loss',factor=0.5,patience=3,\n                                 verbose=1,mode='auto',min_delta=0.005,\n                                 cooldown=5,min_lr=0.0001)\ncallbacks=[reduceLROnPlat]\nmodel.compile(loss='sparse_categorical_crossentropy',optimizer='adam',\n              metrics=['accuracy',top_3_accuracy])\n\nhistory=model.fit(x=x_train,y=y_train,batch_size=32,epochs=20,\n                  validation_data=(x_val,y_val),callbacks=callbacks,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc=history.history['accuracy']\nval_acc=history.history['val_accuracy']\nloss= history.history['loss']\nval_loss=history.history['val_loss']\n\nepochs=range(1,len(acc)+1)\n\nplt.plot(epochs,acc,label='Training acc')\nplt.plot(epochs,val_acc,label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.figure()\n\nplt.plot(epochs,loss,label='Training loss')\nplt.plot(epochs,val_loss,label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list=[]\nreader=pd.read_csv('/kaggle/input/quickdraw-doodle-recognition/test_simplified.csv',index_col=['key_id'],chunksize=2048)\nfor chunk in tqdm(reader,total=55):\n    imagebag=bag.from_sequence(chunk.drawing.values).map(draw_to_img)\n    testarray=np.array(imagebag.compute())\n    testarray=np.reshape(testarray,(testarray.shape[0],height,width,1))\n    testpreds=model.predict(testarray,verbose=0)\n    s=np.argsort(-testpreds)[:,0:3]\n    list.append(s)\narray=np.concatenate(list)\npred_df=pd.DataFrame({'first': array[:,0],'second':array[:,1],'third':array[:,2]})\npred_df=pred_df.replace(nums2names)\npred_df['words']=pred_df['first']+' '+pred_df['second']+' '+pred_df['third']\n\nsub=pd.read_csv('/kaggle/input/quickdraw-doodle-recognition/sample_submission.csv',index_col=['key_id'])\nsub['word']=pred_df.words.values\nsub.to_csv('result_of_mission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}