{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **This notebook shows simple eda about ALL sequence length**  \n## **UPDATE : the data with len is based on newest data as of 20230802, version6**","metadata":{}},{"cell_type":"code","source":"!tree -L 1  /kaggle/input/asl-fingerspelling/","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:51.835059Z","iopub.execute_input":"2023-08-05T00:28:51.835551Z","iopub.status.idle":"2023-08-05T00:28:52.132541Z","shell.execute_reply.started":"2023-08-05T00:28:51.835519Z","shell.execute_reply":"2023-08-05T00:28:52.131415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport re\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:52.134064Z","iopub.execute_input":"2023-08-05T00:28:52.135271Z","iopub.status.idle":"2023-08-05T00:28:53.609316Z","shell.execute_reply.started":"2023-08-05T00:28:52.135211Z","shell.execute_reply":"2023-08-05T00:28:53.607958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"sequence_id 435344989 cannot be found in our dataset, so the cell is filled with nan.","metadata":{}},{"cell_type":"code","source":"train_with_len = pd.read_csv(\"/kaggle/input/asl-fingerspelling-eda/train_with_len.csv\", index_col=0)\ntrain_with_len[\"length\"] = train_with_len[\"length\"].astype(\"int\")\ntrain_with_len = train_with_len.rename(columns={\"length\":\"sequence_length\"})\ntrain_with_len[\"participant_id\"] = train_with_len[\"participant_id\"].astype(\"str\")\ndisplay(train_with_len.head())\nprint(\"---------------------\")\nsupplement_with_len = pd.read_csv(\"/kaggle/input/asl-fingerspelling-eda/supplement_with_len.csv\", index_col=0)\nprint(\"sequence_id 435344989 cannot be found in our dataset\")\ndisplay(supplement_with_len.loc[supplement_with_len[\"sequence_id\"] == 435344989])\n\n# ignore sequence_id 435344989\nsupplement_with_len = supplement_with_len.loc[supplement_with_len[\"sequence_id\"] != 435344989]\nprint(\"---------------------\")\n\nsupplement_with_len[\"length\"] = supplement_with_len[\"length\"].astype(\"int\")\nsupplement_with_len = supplement_with_len.rename(columns={\"length\":\"sequence_length\"})\n\nsupplement_with_len[\"participant_id\"] = supplement_with_len[\"participant_id\"].astype(\"str\")\ndisplay(supplement_with_len.head())","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:53.610646Z","iopub.execute_input":"2023-08-05T00:28:53.613021Z","iopub.status.idle":"2023-08-05T00:28:54.101341Z","shell.execute_reply.started":"2023-08-05T00:28:53.612975Z","shell.execute_reply":"2023-08-05T00:28:54.099995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare with updated data to check data mismatch\n_ = pd.read_csv(\"/kaggle/input/asl-fingerspelling/train.csv\")\n_train = train_with_len.drop(columns=[\"sequence_length\"])\n_train[\"participant_id\"] = _train[\"participant_id\"].astype(\"int\")\n\ntry:\n    print(\"-- check data maching --\")\n    print((_ == _train).sum() == len(_))\nexcept ValueError as e:\n    print(\"different data, should make df_with_len with updated data\")","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.103992Z","iopub.execute_input":"2023-08-05T00:28:54.104371Z","iopub.status.idle":"2023-08-05T00:28:54.243578Z","shell.execute_reply.started":"2023-08-05T00:28:54.104328Z","shell.execute_reply":"2023-08-05T00:28:54.242832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"get phrase_type and phrase_char_len  \nhttps://www.kaggle.com/code/markwijkhuizen/aslfr-eda-preprocessing-dataset","metadata":{}},{"cell_type":"code","source":"# phrase type\n\n\"\"\"\nAttempt to retrieve phrase type\nCould be used for pretraining or type specific inference\n *) Phone Number\\\n *) URL\n *3) Addres\n\"\"\"\ndef get_phrase_type(phrase):\n    # Phone Number\n    if re.match(r'^[\\d+-]+$', phrase):\n        return 'phone_number'\n    # url\n    elif any([substr in phrase for substr in ['www', '.', '/']]) and ' ' not in phrase:\n        return 'url'\n    # Address\n    else:\n        return 'address'\n\ntrain_with_len['phrase_type'] = train_with_len['phrase'].apply(get_phrase_type)\n# supplement_with_len['phrase_type'] = supplement_with_len['phrase'].apply(get_phrase_type) # only sentence","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.244557Z","iopub.execute_input":"2023-08-05T00:28:54.24494Z","iopub.status.idle":"2023-08-05T00:28:54.361721Z","shell.execute_reply.started":"2023-08-05T00:28:54.244903Z","shell.execute_reply":"2023-08-05T00:28:54.360593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# phrase length\ntrain_with_len['phrase_char'] = train_with_len['phrase'].apply(tuple)\ntrain_with_len['phrase_char_length'] = train_with_len['phrase_char'].apply(len)\n\nsupplement_with_len['phrase_char'] = supplement_with_len['phrase'].apply(tuple)\nsupplement_with_len['phrase_char_length'] = supplement_with_len['phrase_char'].apply(len)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.362847Z","iopub.execute_input":"2023-08-05T00:28:54.364833Z","iopub.status.idle":"2023-08-05T00:28:54.492618Z","shell.execute_reply.started":"2023-08-05T00:28:54.364748Z","shell.execute_reply":"2023-08-05T00:28:54.491985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_len","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.493833Z","iopub.execute_input":"2023-08-05T00:28:54.494362Z","iopub.status.idle":"2023-08-05T00:28:54.514077Z","shell.execute_reply.started":"2023-08-05T00:28:54.494334Z","shell.execute_reply":"2023-08-05T00:28:54.513286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplement_with_len","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.515364Z","iopub.execute_input":"2023-08-05T00:28:54.515857Z","iopub.status.idle":"2023-08-05T00:28:54.535665Z","shell.execute_reply.started":"2023-08-05T00:28:54.51583Z","shell.execute_reply":"2023-08-05T00:28:54.534833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Global info","metadata":{}},{"cell_type":"code","source":"train_with_len[\"sequence_length\"].describe()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.53837Z","iopub.execute_input":"2023-08-05T00:28:54.539288Z","iopub.status.idle":"2023-08-05T00:28:54.56116Z","shell.execute_reply.started":"2023-08-05T00:28:54.539266Z","shell.execute_reply":"2023-08-05T00:28:54.558642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_len.loc[train_with_len[\"sequence_length\"] <= 2].head(10)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.565468Z","iopub.execute_input":"2023-08-05T00:28:54.566962Z","iopub.status.idle":"2023-08-05T00:28:54.598643Z","shell.execute_reply.started":"2023-08-05T00:28:54.56689Z","shell.execute_reply":"2023-08-05T00:28:54.597905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_len.loc[train_with_len[\"sequence_length\"] == 807].head(5)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.599712Z","iopub.execute_input":"2023-08-05T00:28:54.600791Z","iopub.status.idle":"2023-08-05T00:28:54.613272Z","shell.execute_reply.started":"2023-08-05T00:28:54.600749Z","shell.execute_reply":"2023-08-05T00:28:54.612567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"/kaggle/input/asl-fingerspelling/\"\ndef get_sequence_length(row):\n    pq_path = os.path.join(data_path, row[\"path\"])\n    print(\"==========\")\n    print(f\"{pq_path = }\")\n    pq_data = pd.read_parquet(pq_path)\n    print(\"sequence_id = \", row[\"sequence_id\"])\n    length = len(pq_data.loc[row[\"sequence_id\"]])\n#     length = len(pq_data[pq_data.index == row[\"sequence_id\"]])\n\n    pq_data = pq_data.loc[row[\"sequence_id\"]]\n    if len(pq_data) == 1630: # error handling \n        pq_data = pd.DataFrame(pq_data).T\n        length = 1\n    display(pq_data)\n    del pq_data; gc.collect()\n    print(f\"{length = }\")\n\n\n    \n# sequence index 48 in train_with_len is really 1 ?? \nget_sequence_length(train_with_len.iloc[48])\n\n# sequence index 517 in train_with_len is really 2 ?? \nget_sequence_length(train_with_len.iloc[517])\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:28:54.614258Z","iopub.execute_input":"2023-08-05T00:28:54.6152Z","iopub.status.idle":"2023-08-05T00:29:09.956522Z","shell.execute_reply.started":"2023-08-05T00:28:54.615172Z","shell.execute_reply":"2023-08-05T00:29:09.955482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplement_with_len[\"sequence_length\"].describe()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:09.957685Z","iopub.execute_input":"2023-08-05T00:29:09.958003Z","iopub.status.idle":"2023-08-05T00:29:09.968669Z","shell.execute_reply.started":"2023-08-05T00:29:09.957976Z","shell.execute_reply":"2023-08-05T00:29:09.968041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplement_with_len.loc[supplement_with_len[\"sequence_length\"] <= 2].head(5)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:09.969508Z","iopub.execute_input":"2023-08-05T00:29:09.970321Z","iopub.status.idle":"2023-08-05T00:29:10.000474Z","shell.execute_reply.started":"2023-08-05T00:29:09.970295Z","shell.execute_reply":"2023-08-05T00:29:09.99873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplement_with_len.loc[supplement_with_len[\"sequence_length\"] == 620].head(5)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:10.002044Z","iopub.execute_input":"2023-08-05T00:29:10.002359Z","iopub.status.idle":"2023-08-05T00:29:10.014721Z","shell.execute_reply.started":"2023-08-05T00:29:10.002333Z","shell.execute_reply":"2023-08-05T00:29:10.013726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sequence index 4399 in supplement_with_len is really 1 ?? \nget_sequence_length(supplement_with_len.iloc[4399])\n\n# sequence index 5062 in supplement_with_len is really 2 ?? \nget_sequence_length(supplement_with_len.iloc[5062])","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:10.015761Z","iopub.execute_input":"2023-08-05T00:29:10.016047Z","iopub.status.idle":"2023-08-05T00:29:45.590858Z","shell.execute_reply.started":"2023-08-05T00:29:10.016025Z","shell.execute_reply":"2023-08-05T00:29:45.589595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train : blue\n# supplement : yellow\nfig, axes = plt.subplots(3, 2, figsize=(8, 8), sharex=\"col\")\nsns.histplot(train_with_len[\"sequence_length\"], ax=axes[0][0], color=\"b\")\nsns.boxplot(train_with_len[\"sequence_length\"], ax=axes[0][1], color=\"b\")\nsns.histplot(supplement_with_len[\"sequence_length\"], ax=axes[1][0], color=\"y\")\nsns.boxplot(supplement_with_len[\"sequence_length\"], ax=axes[1][1], color=\"y\")\nsns.kdeplot(train_with_len[\"sequence_length\"], ax=axes[2][0], color=\"b\")\nsns.kdeplot(supplement_with_len[\"sequence_length\"], ax=axes[2][0], color=\"y\")","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:45.591957Z","iopub.execute_input":"2023-08-05T00:29:45.592194Z","iopub.status.idle":"2023-08-05T00:29:47.48694Z","shell.execute_reply.started":"2023-08-05T00:29:45.592174Z","shell.execute_reply":"2023-08-05T00:29:47.485642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Grouped by participant_id","metadata":{}},{"cell_type":"code","source":"# left : train_with_len\n# right : supplement_with_len\nfig, axes = plt.subplots(1, 2, figsize=(10, 20))\nsns.boxplot(data = train_with_len, x=\"sequence_length\", y=\"participant_id\", ax=axes[0])\nsns.boxplot(data = supplement_with_len, x=\"sequence_length\", y=\"participant_id\", ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:47.488533Z","iopub.execute_input":"2023-08-05T00:29:47.488876Z","iopub.status.idle":"2023-08-05T00:29:50.111393Z","shell.execute_reply.started":"2023-08-05T00:29:47.488847Z","shell.execute_reply":"2023-08-05T00:29:50.110623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# sequence_length vs. phrase_char_length","metadata":{}},{"cell_type":"code","source":"sns.jointplot(data=train_with_len, x=\"sequence_length\", y=\"phrase_char_length\", color=\"b\", hue=\"phrase_type\")\nsns.jointplot(data=train_with_len.loc[train_with_len[\"phrase_type\"]==\"address\"], x=\"sequence_length\", y=\"phrase_char_length\", hue=\"phrase_type\")\nsns.jointplot(data=train_with_len.loc[train_with_len[\"phrase_type\"]==\"url\"], x=\"sequence_length\", y=\"phrase_char_length\", hue=\"phrase_type\")\nsns.jointplot(data=train_with_len.loc[train_with_len[\"phrase_type\"]==\"phone_number\"], x=\"sequence_length\", y=\"phrase_char_length\", hue=\"phrase_type\")\n\n\nsns.jointplot(data=supplement_with_len, x=\"sequence_length\", y=\"phrase_char_length\", color=\"y\")","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:29:50.112425Z","iopub.execute_input":"2023-08-05T00:29:50.112873Z","iopub.status.idle":"2023-08-05T00:30:01.249104Z","shell.execute_reply.started":"2023-08-05T00:29:50.112847Z","shell.execute_reply":"2023-08-05T00:30:01.248086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## sequece_length per character","metadata":{}},{"cell_type":"code","source":"(train_with_len[\"sequence_length\"] / train_with_len[\"phrase_char_length\"]).describe()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:30:01.250512Z","iopub.execute_input":"2023-08-05T00:30:01.250757Z","iopub.status.idle":"2023-08-05T00:30:01.264362Z","shell.execute_reply.started":"2023-08-05T00:30:01.250736Z","shell.execute_reply":"2023-08-05T00:30:01.262975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = (train_with_len[\"sequence_length\"] / train_with_len[\"phrase_char_length\"])\nsns.histplot(data, kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:30:01.268224Z","iopub.execute_input":"2023-08-05T00:30:01.268524Z","iopub.status.idle":"2023-08-05T00:30:02.232388Z","shell.execute_reply.started":"2023-08-05T00:30:01.268489Z","shell.execute_reply":"2023-08-05T00:30:02.230995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data, kde=True)\nplt.xlim(right=30)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:30:02.233689Z","iopub.execute_input":"2023-08-05T00:30:02.233999Z","iopub.status.idle":"2023-08-05T00:30:03.218729Z","shell.execute_reply.started":"2023-08-05T00:30:02.233977Z","shell.execute_reply":"2023-08-05T00:30:03.217997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_len[\"phrase_type\"].value_counts().plot.barh()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:30:03.219848Z","iopub.execute_input":"2023-08-05T00:30:03.220293Z","iopub.status.idle":"2023-08-05T00:30:03.438221Z","shell.execute_reply.started":"2023-08-05T00:30:03.220267Z","shell.execute_reply":"2023-08-05T00:30:03.436285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_len[\"phrase_char_length\"].describe()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:30:56.994786Z","iopub.execute_input":"2023-08-05T00:30:56.995122Z","iopub.status.idle":"2023-08-05T00:30:57.007891Z","shell.execute_reply.started":"2023-08-05T00:30:56.995099Z","shell.execute_reply":"2023-08-05T00:30:57.007191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(train_with_len[\"phrase_char_length\"], kde=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-05T00:31:24.659477Z","iopub.execute_input":"2023-08-05T00:31:24.659982Z","iopub.status.idle":"2023-08-05T00:31:25.240792Z","shell.execute_reply.started":"2023-08-05T00:31:24.659896Z","shell.execute_reply":"2023-08-05T00:31:25.239309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}