{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\nimport json\nimport os\nimport datetime as dt\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def f2cat(filename: str) -> str:\n    return filename.split('.')[0]\n\nclass Simplified():\n    def __init__(self, input_path='./input'):\n        self.input_path = input_path\n\n    def list_all_categories(self):\n        files = os.listdir(os.path.join(self.input_path, 'train_simplified'))\n        return sorted([f2cat(f) for f in files], key=str.lower)\n\n    def read_training_csv(self, category, nrows=None, usecols=None, drawing_transform=False):\n        df = pd.read_csv(os.path.join(self.input_path, 'train_simplified', category + '.csv'),\n                         nrows=nrows, parse_dates=['timestamp'], usecols=usecols)\n        if drawing_transform:\n            df['drawing'] = df['drawing'].apply(json.loads)\n        return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c8edb17435ea500d98aef554b38f551cdc17c93"},"cell_type":"code","source":"start = dt.datetime.now()\ns = Simplified('../input')\nNCSVS = 100\ncategories = s.list_all_categories()\nprint(len(categories))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa103ea464f8b04cce685a3084cdca920c062797"},"cell_type":"code","source":"for y, cat in tqdm(enumerate(categories)):\n    df = s.read_training_csv(cat, nrows=30000)\n    df['y'] = y\n    df['cv'] = (df.key_id // 10 ** 7) % NCSVS\n    for k in range(NCSVS):\n        filename = 'train_k{}.csv'.format(k)\n        chunk = df[df.cv == k]\n        chunk = chunk.drop(['key_id'], axis=1)\n        if y == 0:\n            chunk.to_csv(filename, index=False)\n        else:\n            chunk.to_csv(filename, mode='a', header=False, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13ae737019609901a078648cacdae5f0f10b066f"},"cell_type":"code","source":"for k in tqdm(range(NCSVS)):\n    filename = 'train_k{}.csv'.format(k)\n    if os.path.exists(filename):\n        df = pd.read_csv(filename)\n        df['rnd'] = np.random.rand(len(df))\n        df = df.sort_values(by='rnd').drop('rnd', axis=1)\n        df.to_csv(filename + '.gz', compression='gzip', index=False)\n        os.remove(filename)\nprint(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82113b6b5196ebf453ebebda51ac181a8b904af7"},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Latest run {}.\\nTotal time {}s'.format(end, (end - start).seconds))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}