{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport dask.dataframe as dd\nimport dask.array as da\nfrom dask.diagnostics import ProgressBar\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-13T15:35:55.673715Z","iopub.execute_input":"2023-05-13T15:35:55.674796Z","iopub.status.idle":"2023-05-13T15:35:56.833018Z","shell.execute_reply.started":"2023-05-13T15:35:55.674756Z","shell.execute_reply":"2023-05-13T15:35:56.8313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_dir = '../input/asl-fingerspelling/'\ntrain = pd.read_csv(f'{base_dir}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:35:56.842236Z","iopub.execute_input":"2023-05-13T15:35:56.842615Z","iopub.status.idle":"2023-05-13T15:35:56.956071Z","shell.execute_reply.started":"2023-05-13T15:35:56.842584Z","shell.execute_reply":"2023-05-13T15:35:56.954915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train.csv has the path to each parquet file, the particpant id, sequence_id and sign.\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:35:58.12598Z","iopub.execute_input":"2023-05-13T15:35:58.126393Z","iopub.status.idle":"2023-05-13T15:35:58.143915Z","shell.execute_reply.started":"2023-05-13T15:35:58.126357Z","shell.execute_reply":"2023-05-13T15:35:58.142807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#How to Spell with one Finger an URL? Probably is with more than one :D","metadata":{}},{"cell_type":"code","source":"#By Rob Mulla https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\nfig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"phrase\"].value_counts().head(10).sort_values(ascending=True).plot(\n    kind=\"barh\", ax=ax, title=\"Top 10 Signs in Training Dataset\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-13T15:36:00.28619Z","iopub.execute_input":"2023-05-13T15:36:00.286576Z","iopub.status.idle":"2023-05-13T15:36:00.673326Z","shell.execute_reply.started":"2023-05-13T15:36:00.286546Z","shell.execute_reply":"2023-05-13T15:36:00.672463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/asl-fingerspelling/supplemental_metadata.csv\", delimiter=',', encoding='UTF-8')\npd.set_option('display.max_columns', None)\ndf.head(7)","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:00.914331Z","iopub.execute_input":"2023-05-13T15:36:00.915668Z","iopub.status.idle":"2023-05-13T15:36:01.000114Z","shell.execute_reply.started":"2023-05-13T15:36:00.915625Z","shell.execute_reply":"2023-05-13T15:36:00.998938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"phrase\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:01.538076Z","iopub.execute_input":"2023-05-13T15:36:01.5395Z","iopub.status.idle":"2023-05-13T15:36:01.556613Z","shell.execute_reply.started":"2023-05-13T15:36:01.539453Z","shell.execute_reply":"2023-05-13T15:36:01.555185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"silly = df[(df['phrase']==\"why do you ask silly questions\")].reset_index(drop=True)\nsilly.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:02.306406Z","iopub.execute_input":"2023-05-13T15:36:02.306829Z","iopub.status.idle":"2023-05-13T15:36:02.330652Z","shell.execute_reply.started":"2023-05-13T15:36:02.306795Z","shell.execute_reply":"2023-05-13T15:36:02.329322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#By Rob Mulla https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\nfig, ax = plt.subplots(figsize=(4, 4))\ndf[\"phrase\"].value_counts().head(3).sort_values(ascending=True).plot(\n    kind=\"barh\", color='g', ax=ax, title=\"Top 3 Fingerspelling: Why do you ask silly questions?\"\n)\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-13T15:36:02.9792Z","iopub.execute_input":"2023-05-13T15:36:02.97964Z","iopub.status.idle":"2023-05-13T15:36:03.223272Z","shell.execute_reply.started":"2023-05-13T15:36:02.979595Z","shell.execute_reply":"2023-05-13T15:36:03.222192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_fn = train.query('phrase == \"hentaihubs.com\"')[\"path\"].values[0]\n\nex_landmark = pd.read_parquet(f\"{base_dir}/{ex_fn}\")\nex_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:18.615842Z","iopub.execute_input":"2023-05-13T15:36:18.616283Z","iopub.status.idle":"2023-05-13T15:36:35.66793Z","shell.execute_reply.started":"2023-05-13T15:36:18.616247Z","shell.execute_reply":"2023-05-13T15:36:35.666674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = ex_landmark[\"frame\"].nunique()\n#unique_types = example_landmark[\"type\"].nunique()\n#types_in_video = example_landmark[\"type\"].unique()\nprint(\n    f\"The file has {unique_frames} unique frames\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:39.871295Z","iopub.execute_input":"2023-05-13T15:36:39.871998Z","iopub.status.idle":"2023-05-13T15:36:39.882736Z","shell.execute_reply.started":"2023-05-13T15:36:39.871958Z","shell.execute_reply":"2023-05-13T15:36:39.881533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Really? 672 unique frames. I've No clue how to deal with that ","metadata":{}},{"cell_type":"markdown","source":"#Read only one file. At least.","metadata":{}},{"cell_type":"code","source":"ddf = dd.read_parquet(\"/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:44.487139Z","iopub.execute_input":"2023-05-13T15:36:44.487541Z","iopub.status.idle":"2023-05-13T15:36:44.89412Z","shell.execute_reply.started":"2023-05-13T15:36:44.487501Z","shell.execute_reply":"2023-05-13T15:36:44.892884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlen(ddf)","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:46.274212Z","iopub.execute_input":"2023-05-13T15:36:46.274632Z","iopub.status.idle":"2023-05-13T15:36:46.824224Z","shell.execute_reply.started":"2023-05-13T15:36:46.274595Z","shell.execute_reply":"2023-05-13T15:36:46.82294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ddf.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:36:46.978231Z","iopub.execute_input":"2023-05-13T15:36:46.97866Z","iopub.status.idle":"2023-05-13T15:37:10.226169Z","shell.execute_reply.started":"2023-05-13T15:36:46.978624Z","shell.execute_reply":"2023-05-13T15:37:10.224739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ddf.x_face_0.describe().compute()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:37:10.228237Z","iopub.execute_input":"2023-05-13T15:37:10.228628Z","iopub.status.idle":"2023-05-13T15:37:10.346196Z","shell.execute_reply.started":"2023-05-13T15:37:10.228595Z","shell.execute_reply":"2023-05-13T15:37:10.345217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/luxcem/dask-on-large-parquet-dataset/notebook\n\nddf.where(ddf.x_face_0 > 50).visualize()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:37:10.347231Z","iopub.execute_input":"2023-05-13T15:37:10.347577Z","iopub.status.idle":"2023-05-13T15:37:10.937515Z","shell.execute_reply.started":"2023-05-13T15:37:10.347548Z","shell.execute_reply":"2023-05-13T15:37:10.936192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/luxcem/dask-on-large-parquet-dataset/notebook\n\nh, bins = da.histogram(ddf[[\"x_face_0\"]].to_dask_array(), bins=(0, 1, 2, 5, 10, 20, 50, 100, 1000))","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:37:10.940144Z","iopub.execute_input":"2023-05-13T15:37:10.940532Z","iopub.status.idle":"2023-05-13T15:37:10.953714Z","shell.execute_reply.started":"2023-05-13T15:37:10.940498Z","shell.execute_reply":"2023-05-13T15:37:10.952506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ProgressBar():\n    hd = h.compute()\n    bins = bins.compute()","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:37:10.954984Z","iopub.execute_input":"2023-05-13T15:37:10.955326Z","iopub.status.idle":"2023-05-13T15:37:19.24811Z","shell.execute_reply.started":"2023-05-13T15:37:10.955295Z","shell.execute_reply":"2023-05-13T15:37:19.246785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/luxcem/dask-on-large-parquet-dataset/notebook\nplt.style.use('fivethirtyeight')\nsns.barplot(x=bins[1:], y=hd);","metadata":{"execution":{"iopub.status.busy":"2023-05-13T15:37:19.249626Z","iopub.execute_input":"2023-05-13T15:37:19.250187Z","iopub.status.idle":"2023-05-13T15:37:19.567876Z","shell.execute_reply.started":"2023-05-13T15:37:19.250151Z","shell.execute_reply":"2023-05-13T15:37:19.566899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}