{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c253b65f6416210af591b5797e4af71d4aa3b0c4"},"cell_type":"markdown","source":"### This code reads samples from all 340 categories csv files and prepare 100 (=NCSVS) shuffled compressed csv for easy read in main code.\n\n#### Points to note:\n1. In general each category csv has total of more than 100K rows, out of which a sample of 35K rows are read. \n2. only recognised (=True) has been taken from sample of 35K\n3. all 100 csvs formed from 340 csvs may not have same #rows."},{"metadata":{"trusted":true,"_uuid":"f54295663b07c20dccd519c7f389caea4d918bb4"},"cell_type":"code","source":"import os\nimport numpy as np\nfrom tqdm import tqdm\nimport pandas as pd\nfrom sklearn.utils import shuffle","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"SAMPLE_SIZE_PER_CSV = 35000\nPIXELS = 64\nNCSVS = 100\nALL_TRAIN_CSV = os.listdir(os.path.join(\"../input/train_simplified/\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7112c6602c2e9aff1a0d1fb40e033b59bf139c87"},"cell_type":"code","source":"categories = [word.split('.')[0] for word in ALL_TRAIN_CSV]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab418e135e9185b249b3292562b28931d95002a9"},"cell_type":"code","source":"for y, cat in tqdm(enumerate(categories)):\n    df = pd.read_csv(os.path.join(\"../input/train_simplified/\", cat + '.csv'),nrows=SAMPLE_SIZE_PER_CSV)\n    df = df[df['recognized']==True][['drawing','word']]\n    df['y'] = y\n    rnd_index = list(np.arange(NCSVS))*(int(len(df)/NCSVS)+1)\n    df['csv_index'] = rnd_index[:len(df)]\n    for k in range(NCSVS):\n        filename = 'train_k{}.csv'.format(k)\n        chunk = df[df.csv_index == k]\n        chunk = chunk.drop(['csv_index'], axis=1)\n        if y == 0:\n            chunk.to_csv(filename, index=False)\n        else:\n            chunk.to_csv(filename, mode='a', header=False, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"182890b0495a5046444d642fdbcc9a84f681bc13"},"cell_type":"code","source":"for k in tqdm(range(NCSVS)):\n    filename = 'train_k{}.csv'.format(k)\n    if os.path.exists(filename):\n        df = pd.read_csv(filename)\n        df = shuffle(df)\n        df.to_csv(filename + '.gz', compression='gzip', index=False)\n        os.remove(filename)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ccbd1aeb9c4301c68e7dc99e0099715eb0f85df"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}