{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport ast\nimport os\nimport datetime as dt\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\nclass Simplified():\n    def __init__(self, input_path='./input'):\n        self.input_path = input_path\n\n    def list_all_categories(self):\n        files = os.listdir(os.path.join(self.input_path, 'train_simplified'))\n        return sorted([f2cat(f) for f in files], key=str.lower)\n\n    def read_training_csv(self, category, nrows=None, usecols=None, drawing_transform=False):\n        df = pd.read_csv(os.path.join(self.input_path, 'train_simplified', category + '.csv'),\n                         nrows=nrows, parse_dates=['timestamp'], usecols=usecols)\n        if drawing_transform:\n            df['drawing'] = df['drawing'].apply(ast.literal_eval)\n        return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c8edb17435ea500d98aef554b38f551cdc17c93"},"cell_type":"code","source":"start = dt.datetime.now()\ns = Simplified('../input')\nNCSVS = 100\ncategories = s.list_all_categories()\nprint(len(categories))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6b6a3aab3f3717e075de4042b29eab5b4759008"},"cell_type":"code","source":"animals = ['ant', 'bat', 'bear', 'bee', 'bird', 'butterfly', 'camel', 'cat', 'cow',\n           'crab', 'crocodile', 'dog', 'dolphin', 'dragon', 'duck', 'elephant', 'fish',\n           'flamingo', 'frog', 'giraffe', 'hedgehog', 'horse', 'kangaroo', 'lion',\n           'lobster', 'monkey', 'mosquito', 'mouse', 'octopus', 'owl', 'panda',\n           'parrot', 'penguin', 'pig', 'rabbit', 'raccoon', 'rhinoceros', 'scorpion',\n           'sea turtle', 'shark', 'sheep', 'snail', 'snake', 'spider', 'squirrel',\n           'swan', 'teddy-bear', 'tiger', 'whale', 'zebra']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa103ea464f8b04cce685a3084cdca920c062797"},"cell_type":"code","source":"for y, cat in tqdm(enumerate(animals)):\n    df = s.read_training_csv(cat, nrows=100000)\n    df['y'] = y\n    df['cv'] = (df.key_id // 10 ** 7) % NCSVS\n    for k in range(NCSVS):\n        filename = 'train_k{}.csv'.format(k)\n        chunk = df[df.cv == k]\n        chunk = chunk.drop(['key_id'], axis=1)\n        if y == 0:\n            chunk.to_csv(filename, index=False)\n        else:\n            chunk.to_csv(filename, mode='a', header=False, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13ae737019609901a078648cacdae5f0f10b066f"},"cell_type":"code","source":"for k in tqdm(range(NCSVS)):\n    filename = 'train_k{}.csv'.format(k)\n    if os.path.exists(filename):\n        df = pd.read_csv(filename)\n        df['rnd'] = np.random.rand(len(df))\n        df = df.sort_values(by='rnd').drop('rnd', axis=1)\n        df.to_csv(filename + '.gz', compression='gzip', index=False)\n        os.remove(filename)\nprint(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82113b6b5196ebf453ebebda51ac181a8b904af7"},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Latest run {}.\\nTotal time {}s'.format(end, (end - start).seconds))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}