{"cells":[{"metadata":{"trusted":true,"_uuid":"add17ab04310a2aad36231cf97c8971d667457b9"},"cell_type":"code","source":"!pip3 install git+https://github.com/fastai/fastai.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d74054b8352428daeb0ef067cfc9a3c3958f539"},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\n\nfrom fastai import *\nfrom fastai.vision import *\n\nimport json\n\nfrom utils import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87268d8ba4c423dc5cea943381ce49da9e8dd192"},"cell_type":"code","source":"NUM_SAMPLES_PER_CLASS = 1_000\nNUM_VAL = 50 * 340","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4b7a265215e9667c8d008294d50213b9bf42d8f"},"cell_type":"code","source":"!ls ../input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df0f63a1a4217a6489237e1597224d86228dbcb4"},"cell_type":"code","source":"PATH = Path('/kaggle/working/data')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce49aef0383418066efe26ec5d419d5624b2c85c"},"cell_type":"markdown","source":"### Generate data from csvs"},{"metadata":{"trusted":true,"_uuid":"8a48235da42a1dc16e26196ccfbb305f31da79ff"},"cell_type":"code","source":"PATH.mkdir(exist_ok=True)\n(PATH/'train').mkdir(exist_ok=True)\n(PATH/'test').mkdir(exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b06e1d09f67f5da041c2a28e2146f0f7299dfb41"},"cell_type":"code","source":"def create_train_txts_from_df(path):\n    df = pd.read_csv(path)\n    klass = '_'.join(path.stem.split())\n    (PATH/'train'/klass).mkdir(exist_ok=True)\n    for row in df.sample(NUM_SAMPLES_PER_CLASS).iterrows():\n        example = {\n            'countrycode': row[1].countrycode,\n            'drawing': json.loads(row[1].drawing),\n            'key_id': row[1].key_id,\n            'recognized': row[1].recognized\n        }\n        with open(PATH/'train'/klass/f'{example[\"key_id\"]}.txt', mode='w') as f: json.dump(example, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc488de81364031666194b3313c721989b6a3620"},"cell_type":"code","source":"%time for p in Path('../input/train_simplified').iterdir(): create_train_txts_from_df(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"583f495b4f3a3483dd081e97b4138532039d3f6c"},"cell_type":"code","source":"def create_test_txts_from_df(path):\n    print(path)\n    df = pd.read_csv(path)\n    for row in df.iterrows():\n        example = {\n            'countrycode': row[1].countrycode,\n            'drawing': json.loads(row[1].drawing),\n            'key_id': row[1].key_id\n        }\n        with open(PATH/'test'/f'{example[\"key_id\"]}.txt', mode='w') as f: json.dump(example, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73b0ec205b1f1c6218714ed3f6dce098be4f2eeb"},"cell_type":"code","source":"%time create_test_txts_from_df('../input/test_simplified.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"114d508ed79807f6c9c7342a32d337f2a3cc6695"},"cell_type":"markdown","source":"###  Create ImageDataBunch using the data block API"},{"metadata":{"trusted":true,"_uuid":"35959a97d1a2219c99dce470d2ee2ece672fce7b"},"cell_type":"code","source":"sz = 128\nbs = 640","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed63fca58771d707e1b06115a7eeaa92db4d4e96"},"cell_type":"code","source":"def create_func(path):\n    with open(path) as f: j = json.load(f)\n    drawing = list2drawing(j['drawing'], size=sz)\n    tensor = drawing2tensor(drawing)\n    return Image(tensor.div_(255))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b91f6441f8013771e3bee77bd82ba88331304d7"},"cell_type":"code","source":"item_list = ItemList.from_folder(PATH/'train', create_func=create_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a965741bb2fcbcf2641f89beebd61e36c130c369"},"cell_type":"code","source":"idxs = np.arange(item_list.items.shape[0])\nnp.random.shuffle(idxs)\nval_idxs = idxs[:NUM_VAL]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5537b5db6c199771043c4b261d0a00159cd9084e"},"cell_type":"code","source":"item_lists = item_list.split_by_idx(val_idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3de92f2003b8e89e867f0b1d3ec4e545242f249"},"cell_type":"code","source":"label_lists = item_lists.label_from_folder()\npd.to_pickle(label_lists.train.y.classes, '/kaggle/working/classes.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dea5d7e65a27ecd6d7983c7ad35adb19144a7f51"},"cell_type":"code","source":"classes = pd.read_pickle('../input/classes.pkl')\n\nlabel_lists = item_lists.label_from_folder(classes=classes)\n\ntest_items = ItemList.from_folder(PATH/'test', create_func=create_func)\nlabel_lists.add_test(test_items);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41306deb6866ba4804c94c848b140a892f8cba69"},"cell_type":"code","source":"train_dl = DataLoader(label_lists.train, bs, True, num_workers=12)\nvalid_dl = DataLoader(label_lists.valid, 2*bs, False, num_workers=12)\ntest_dl = DataLoader(label_lists.test, 2*bs, False, num_workers=12)\n\ndata_bunch = ImageDataBunch(train_dl, valid_dl, test_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edcb0e24a22a75b1cc04a6b7b5ad3221b9524ba0"},"cell_type":"code","source":"# pd.to_pickle(data_bunch.batch_stats(), f'data/batch_stats_{sz}.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2d258b261321877d4eab4086ebe1076dcb088a1"},"cell_type":"code","source":"batch_stats = pd.read_pickle(f'data/batch_stats_{sz}.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4ccbf1c7490c1ae2dbd74904e2412eb76e7b4f0"},"cell_type":"code","source":"data_bunch.normalize(batch_stats);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90e5d5613db5e1cbec6711bb83875dcab53fa9b3"},"cell_type":"code","source":"data_bunch.show_batch(rows=4)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27778f40c76ccf4ca1c2bc7b03dcdb5cfbf2e043"},"cell_type":"markdown","source":"### Train"},{"metadata":{"trusted":true,"_uuid":"97e39d3c744b4ca4af2a048c5811c0a41f6488ec"},"cell_type":"code","source":"name = f'res34-{sz}'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"678c2f1eff43e75650f45fdf59238e9184e84383"},"cell_type":"code","source":"learn = create_cnn(data_bunch, models.resnet34, metrics=[accuracy, map3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d38d056412c1ecdc8b82082826954dbc56bd99d"},"cell_type":"code","source":"learn.fit_one_cycle(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"261cfec42a3090c65abaf760f76dd2f647830e14"},"cell_type":"code","source":"learn.save(f'{name}-stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce30fad4e89afa296c4aa50b421222cc9b1d74ec"},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe0f9bb92cbc5232d1733cc9c55f5db258bf5a1"},"cell_type":"code","source":"learn.fit_one_cycle(6, max_lr=6e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db6703ef134721f1d05d647d5f827c23dd4a0c3f"},"cell_type":"code","source":"learn.save(f'{name}-stage-2')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78402ba96f9e57ba743e1d88ac954229144c0623"},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true,"_uuid":"3cd07d16c5c2cb61397ae8981a09b620bb284449"},"cell_type":"code","source":"learn.load(f'{name}-stage-2');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04836cdbd3e634972051f2e93a8fb813747d7bc6"},"cell_type":"code","source":"preds, _ = learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"673fddbb79325b068da47118f6c5d6e991cd6f43"},"cell_type":"code","source":"create_submission(preds, data_bunch.test_dl, name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb16c3590079cc227b376cdf55437697462ac894"},"cell_type":"code","source":"pd.read_csv(f'subs/{name}.csv.gz').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d26c622c3a9af0d84b9498e6cb90935360b7203d"},"cell_type":"code","source":"# !kaggle competitions submit -c quickdraw-doodle-recognition -f subs/{name}.csv.gz -m \"{name}\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}