{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport ast\nimport datetime\nimport random\nimport joblib\nfrom dask import delayed\nimport dask.dataframe as dd\n\nimport plotly.plotly as py\nimport plotly.graph_objs as go\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nnp.warnings.filterwarnings('ignore')\nimport plotly.figure_factory as ff\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\ninit_notebook_mode(connected=True)\nimport cufflinks as cf\ncf.set_config_file(offline=True, world_readable=True, theme='ggplot')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6df004f058808f6e199a2cbed21a50b2ff4d91f8","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train_filepaths = glob.glob('../input/train_simplified/*.csv')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true,"_uuid":"7f0319cd62e7dd73d0bb9a69e49cfc9f35819c88"},"cell_type":"code","source":"train_filepaths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87239f7292918a18c0ce186c817cfd02a686cf75"},"cell_type":"code","source":"# change this set to compare different classes\nselected_classes = {'sleeping bag', 'house', 'dragon', 'The Mona Lisa', 'star', 'lightning'}\nselected_fps = [f'../input/train_simplified/{x}.csv' for x in selected_classes]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"217d733db7c3607cbdbd976a420d232017097416"},"cell_type":"code","source":"def make_df(fp):\n    dtypes = {\n    'countrycode': 'category',\n    'drawing': np.str,\n    'key_id': np.uint64,\n    'recognized': np.bool_,\n    'timestamp': np.str,\n    'word': 'category',\n    }\n    df = pd.read_csv(fp, dtype=dtypes)\n    df['word'] = df['word'].replace(' ', '_', regex=True)\n    df[\"drawing\"] = df[\"drawing\"].apply(lambda x: ast.literal_eval(x))\n    df[\"timestamp\"] = df[\"timestamp\"].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S.%f'))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"9655fd36918c2ea766109498fa73b98b22dc8ed2"},"cell_type":"code","source":"def plot_drawing(drawing, ax, **kwargs):\n    for line_set in drawing:\n        xs, ys = line_set\n        ax.plot(xs, ys, **kwargs)\n\ndef overlay_drawings(drawings, ax, **kwargs):\n    for drawing in drawings:\n        plot_drawing(drawing, ax, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9b7143da0f4dcb347d211b0046d35791b137569"},"cell_type":"code","source":"def plot_overlays(df, sample_size=100, **kwargs):\n    fig, axes = plt.subplots(1, 2, figsize=(14, 8))\n    n_rec = df['recognized'].sum()\n    n_not_rec = (df['recognized'] == False).sum()\n    n_rec = n_rec if sample_size > n_rec else sample_size\n    n_not_rec = n_not_rec if sample_size > n_not_rec else sample_size\n    \n    overlay_drawings(df[df['recognized']]['drawing'].sample(n_rec).tolist(), axes[0], **kwargs)\n    axes[0].set_title('recognized', fontsize=14)\n    overlay_drawings(df[~df['recognized']]['drawing'].sample(n_not_rec).tolist(), axes[1], **kwargs)\n    axes[1].set_title('not recognized', fontsize=14)\n    \n    fig.suptitle(df.word[0], y=1.02, fontsize=14)\n    fig.tight_layout()\n    axes[0].invert_yaxis()\n    axes[1].invert_yaxis()\n    axes[0].axis('off')\n    axes[1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"9eef1836d7dd298d458990388b329ef3cd571fb4"},"cell_type":"code","source":"selected_dfs = joblib.Parallel(n_jobs=4)(joblib.delayed(make_df)(fp) for fp in selected_fps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"61c148c017cfb533abb6083fcf675087abe1147f"},"cell_type":"code","source":"# plot some sample drawings for each word\nn_samples = 4\nfor df in selected_dfs:\n    \n    fig, axes = plt.subplots(1, n_samples, figsize=(6*n_samples, 6))\n    for ax in axes:\n        sample_df = df.sample(1)\n        plot_drawing(sample_df.drawing.tolist()[0], ax)\n        ax.invert_yaxis()\n        ax.axis('off')\n        rec = sample_df.recognized.tolist()[0]\n        rec_text = \"recognized\" if sample_df.recognized.tolist()[0] else \"not recognized\"\n        ax.set_title(rec_text, fontsize=12)\n    fig.suptitle(df.word[0], y=1.02, fontsize=14)\n    fig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"8735647c2e447aa7434dffacfe1c00bb09d8ba89"},"cell_type":"code","source":"# plot many samples overlaid ontop of each other for each word (separating recognized vs not recognized into two plots)\nn_samples = 500\nfor df in selected_dfs:\n    plot_overlays(df, n_samples, linewidth=5, alpha=.01, color='blue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbd722bec9939978f2d041886ec7092fc2693175"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a1307211ae3933ca014ca8b46213aa08e3de781"},"cell_type":"code","source":"def make_df_summary(fp):\n    dtypes = {\n    'countrycode': 'category',\n    'drawing': np.str,\n    'key_id': np.uint64,\n    'recognized': np.bool_,\n    'timestamp': np.str,\n    'word': 'category',\n    }\n    df = pd.read_csv(fp, dtype=dtypes)\n    df['word'] = df['word'].replace(' ', '_', regex=True).astype('category')\n    df[\"drawing\"] = df[\"drawing\"].apply(lambda x: ast.literal_eval(x))\n    df[\"number_lines\"] = df[\"drawing\"].apply(lambda x: len(x)).astype(np.uint16)\n    df[\"mean_points_in_line\"] = df[\"drawing\"].apply(lambda x: np.mean([len(line[0]) for line in x])).astype(np.float32)\n    df[\"timestamp\"] = df[\"timestamp\"].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S.%f'))\n    df = df.drop([\"drawing\", \"key_id\"], axis='columns')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"dc0e9a0eed2afc876abfa753099677910032d238"},"cell_type":"code","source":"dfs = joblib.Parallel(n_jobs=4)(joblib.delayed(make_df_summary)(fp) for fp in selected_fps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4d99e5af38d447ff1030ca0b6a9958cd26bd605"},"cell_type":"code","source":"df = dd.concat(dfs, axis=0, interleave_partitions=True)\ndf = df.compute()\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61305bcf79126f97de148d798794b2b04cdab3fd"},"cell_type":"code","source":"def plot_by_word_dists(df, feature):\n    words = [word for word in df.word.unique()]\n    hist_data = [df[df['word']==word][feature] for word in words]\n    fig = ff.create_distplot(hist_data, words, show_curve=False, show_rug=False)\n    fig['layout'].update(xaxis=dict(title=feature), yaxis=dict(title='Frequency'))\n    iplot(fig, filename=f'{feature}_plot')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"407b96c61f0fa21733342f410347d91689abb36c"},"cell_type":"code","source":"plot_by_word_dists(df, feature='number_lines')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45a5ede3589f894c1ba1a65bb1ccfc1e0cc91d95","scrolled":false},"cell_type":"code","source":"plot_by_word_dists(df, feature='mean_points_in_line')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bc3d05f522a98ec960e00ec8c99a473c079d6b6"},"cell_type":"code","source":"def plot_recognized_vs_not_recognized_by_word(df):\n    agg_df = df.groupby('word').agg({'recognized': ['sum', 'size']})\n    agg_df.columns = agg_df.columns.map('_'.join)\n    agg_df = agg_df.rename(columns={'recognized_sum': 'recognized', 'recognized_size': 'size'})\n    agg_df['not_recognized'] = agg_df['size'] - agg_df['recognized']\n    agg_df['not_recognized'] /= agg_df['size']\n    agg_df['recognized'] /= agg_df['size']\n    agg_df.index = agg_df.index.astype(np.str)\n    print(agg_df.head())\n    agg_df.iplot(kind='bar', keys=['recognized', 'not_recognized'], filename='recognized_vs_not', xTitle='Word', yTitle='Frequency', title='Recognized vs Not Recognized by Word')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72be84e38528e84f1e3abf3f43895524054b03b6"},"cell_type":"code","source":"plot_recognized_vs_not_recognized_by_word(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45ffd70596aa791aba3801a0eeed2534b905808c"},"cell_type":"code","source":"def plot_count_by_word(df):\n    agg_df = df.groupby('word').agg({'recognized': ['sum', 'size']})\n    agg_df.columns = agg_df.columns.map('_'.join)\n    agg_df = agg_df.rename(columns={'recognized_sum': 'recognized', 'recognized_size': 'size'})\n    agg_df.index = agg_df.index.astype(np.str)\n    agg_df['size'].iplot(kind='bar', filename='samples_per_word', xTitle='Word', yTitle='Count', title='Number of Samples per Word')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b320e798d0331dad46d7df634586eb30cf447f3d"},"cell_type":"code","source":"plot_count_by_word(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7097753d20420acafee0682b8d294414230c8a1"},"cell_type":"code","source":"def plot_count_by_word_by_country(df, country_cols=None):\n    if not country_cols:\n        country_cols = ['US', 'GB', 'CA', 'DE', 'AU', 'RU', 'FI', 'BR', 'KR']\n    \n    agg_df = df.groupby(['word', 'countrycode']).size()\n    agg_df.name = 'count'\n    agg_df = agg_df.reset_index(drop=False)\n\n    words = list(agg_df.word.unique())\n    countrycodes = list(agg_df.countrycode.unique())\n\n    records = []\n    for word in agg_df.word.unique():\n        counts_for_countries = {x: 0 for x in countrycodes}\n        word_df = agg_df[agg_df.word == word]\n        dicts = word_df[['countrycode', 'count']].to_dict(orient='records')\n        present_counts_for_countries = {d['countrycode']: d['count'] for d in dicts}\n        counts_for_countries.update(present_counts_for_countries)\n        counts_for_countries['word'] = word\n        records.append(counts_for_countries)\n        \n    df = pd.DataFrame(records)\n    df.index = df.pop('word')\n    \n    df[country_cols].iplot(kind='bar', filename='samples_per_word_per_country', xTitle='Word', yTitle='Count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e23544b7457f1221a904d529ce914d8146aeda25"},"cell_type":"code","source":"plot_count_by_word_by_country(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d574efa12ffd6d3ab87817300b9b6119ac347bc4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}