{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#-*- encoding: utf8 -*-\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport json\nimport zipfile\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\nimport ast\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir ../working/main_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"./","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 데이터 구성 분석\n* drawing is consisit of [x,y] point and start, end 2d 형식으로 구성\n* recognized 는 해당 drawing에 대한 진위 여부"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"SEED = 42\nINPUT_DIR = \"../input/\"\ntrain_simple_eiffel = pd.read_csv(os.path.join(INPUT_DIR, 'train_simplified/The Eiffel Tower.csv'))\ntrain_simple_eiffel.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_simple_eiffel['recognized'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Matplot 시각화를 통한 데이터 형태 분석\n* Line이 작성된 순서에 따른 Sequential한 성질을 보임(Sequential 특징이 반영된 모델 추가 구성)\n* recognized False 케이스는 대체로 관련성 없는 형태의 그림을 보임\n* Evaluate Metric이 Precision에 집중된 Task, 따라서 True에 해당하는 데이터를 중심으로 학습 진행\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_simple_eiffel['drawing'] = train_simple_eiffel['drawing'].apply(ast.literal_eval)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 10\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(16, 10))\nfor i, drawing in enumerate(train_simple_eiffel[train_simple_eiffel['recognized']==True][:100].drawing):\n    ax = axs[i // n, i % n]\n    for x, y in drawing:\n        ax.plot(x, -np.array(y), lw=3)\n    ax.axis('off')\nfig.savefig('eiffe_true.png', dpi=200)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 10\nfig, axs = plt.subplots(nrows=n, ncols=n, sharex=True, sharey=True, figsize=(16, 10))\nfor i, drawing in enumerate(train_simple_eiffel[train_simple_eiffel['recognized']==False][:100].drawing):\n    ax = axs[i // n, i % n]\n    for x, y in drawing:\n        ax.plot(x, -np.array(y), lw=3)\n    ax.axis('off')\nfig.savefig('eiffe_false.png', dpi=200)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train,Test data 생성\n* 340개의 카테고리를 KEYID 기준으로 Split을 통하여 100개의 categories mixed 데이터셋 생성\n* recognized True인 데이터 셋만 가져옴"},{"metadata":{"trusted":true},"cell_type":"code","source":"CHUNK_DIR = '../working/main_data/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(seed = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir(os.path.join(INPUT_DIR, 'train_simplified/')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir(os.path.join(INPUT_DIR, 'train_simplified/'))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def file2cat(filename):\n    return filename.split('.')[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Simplified():\n    def __init__(self, input_path = '../input'):\n        self.input_path = input_path\n    def list_all_categories(self):\n        files = os.listdir(os.path.join(self.input_path, 'train_simplified'))\n        return sorted([file2cat(f) for f in files], key = str.lower)\n    def read_training_csv(self, category, nrows = None, usecols = None, drawing_transform = False):\n        df = pd.read_csv(os.path.join(self.input_path, 'train_simplified', category + '.csv'),\\\n                        parse_dates = ['timestamp'],usecols=usecols)\n        df = df[df['recognized']==True].reset_index(drop=True)\n        df = df.iloc[:nrows]\n        \n        if drawing_transform:\n            df['drawing'] = df['drawing'].apply(json.loads)\n        return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s = Simplified(INPUT_DIR)\nDATA_CHUNK = 100\ncategories = s.list_all_categories()\nprint(len(categories))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for y, cat in tqdm(enumerate(categories)):\n    df = s.read_training_csv(cat, nrows=30000)\n    df['y'] = y\n    df['cv'] = (df.key_id//10 ** 7) % DATA_CHUNK\n    for k in range(DATA_CHUNK):\n        filename = 'train_k{}.csv'.format(k)\n        filename = os.path.join(CHUNK_DIR, filename)\n        chunk = df[df.cv==k]\n        chunk = chunk.drop(['key_id'],axis = 1)\n        if y==0:\n            chunk.to_csv(filename, index =False)\n        else:\n            chunk.to_csv(filename, mode = 'a', header = False, index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nfrom multiprocessing import Pool\nfrom functools import partial\nfrom itertools import repeat\nfrom itertools import product","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multiprocessing을 통한 병렬처리"},{"metadata":{"trusted":true},"cell_type":"code","source":"!grep -c processor /proc/cpuinfo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NJOBS = !grep -c processor /proc/cpuinfo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def chunk2gzip(chunk_number):\n#     for k in range(chunk_number,JOBS):\n#         try:\n    filename = 'train_k{}.csv'.format(chunk_number)\n    filename = os.path.join(CHUNK_DIR, filename)\n    if os.path.exists(filename):\n        df = pd.read_csv(filename)\n        print(df.shape)\n        df['rnd'] = np.random.rand(len(df))\n        df = df.sort_values(by ='rnd').drop('rnd', axis =1)\n        df.to_csv(filename + '.gz', compression='gzip', index = False)\n        os.remove(filename)\n#         except IndexError:\n#             print(\"file no. {} not in\".format(k))\n#             pass\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with Pool(processes=int(NJOBS[0])) as p:\n    max_ = DATA_CHUNK\n    with tqdm(total=max_) as pbar:\n        for i, _ in tqdm(enumerate(p.imap(chunk2gzip, list(range(0,100))))):\n            pbar.update()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}