{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Supplemental Metadata Overview","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\nimport plotly.io as pio\nimport warnings\nwarnings.filterwarnings('ignore ')","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:31.716459Z","iopub.execute_input":"2023-05-14T08:39:31.716884Z","iopub.status.idle":"2023-05-14T08:39:32.31406Z","shell.execute_reply.started":"2023-05-14T08:39:31.71684Z","shell.execute_reply":"2023-05-14T08:39:32.312931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sm_df = pd.read_csv('/kaggle/input/asl-fingerspelling/supplemental_metadata.csv')\nsm_df","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:32.316248Z","iopub.execute_input":"2023-05-14T08:39:32.316727Z","iopub.status.idle":"2023-05-14T08:39:32.413888Z","shell.execute_reply.started":"2023-05-14T08:39:32.316694Z","shell.execute_reply":"2023-05-14T08:39:32.412685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_id = 1535467051\nfile_id = 33432165","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:32.415268Z","iopub.execute_input":"2023-05-14T08:39:32.41563Z","iopub.status.idle":"2023-05-14T08:39:32.421099Z","shell.execute_reply.started":"2023-05-14T08:39:32.415574Z","shell.execute_reply":"2023-05-14T08:39:32.420264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_one_parquet_data = pd.read_parquet(f'/kaggle/input/asl-fingerspelling/supplemental_landmarks/{file_id}.parquet', engine='pyarrow')\nread_one_parquet_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:32.423231Z","iopub.execute_input":"2023-05-14T08:39:32.423753Z","iopub.status.idle":"2023-05-14T08:39:36.435199Z","shell.execute_reply.started":"2023-05-14T08:39:32.423722Z","shell.execute_reply":"2023-05-14T08:39:36.433367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_one_parquet_data.columns","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:36.436266Z","iopub.execute_input":"2023-05-14T08:39:36.436574Z","iopub.status.idle":"2023-05-14T08:39:36.444724Z","shell.execute_reply.started":"2023-05-14T08:39:36.436545Z","shell.execute_reply":"2023-05-14T08:39:36.443666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_one_parquet_data.index == 435344989","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:36.446348Z","iopub.execute_input":"2023-05-14T08:39:36.446995Z","iopub.status.idle":"2023-05-14T08:39:36.456236Z","shell.execute_reply.started":"2023-05-14T08:39:36.446915Z","shell.execute_reply":"2023-05-14T08:39:36.455171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_supp = read_one_parquet_data[read_one_parquet_data.index == sequence_id]\nprint(sequence_supp.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:36.457752Z","iopub.execute_input":"2023-05-14T08:39:36.458162Z","iopub.status.idle":"2023-05-14T08:39:36.467404Z","shell.execute_reply.started":"2023-05-14T08:39:36.458125Z","shell.execute_reply":"2023-05-14T08:39:36.466191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_one_parquet_data = read_one_parquet_data.dropna().reset_index(drop=True)\nread_one_parquet_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:36.469005Z","iopub.execute_input":"2023-05-14T08:39:36.469421Z","iopub.status.idle":"2023-05-14T08:39:37.016738Z","shell.execute_reply.started":"2023-05-14T08:39:36.469383Z","shell.execute_reply":"2023-05-14T08:39:37.015644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_one_parquet_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:37.018118Z","iopub.execute_input":"2023-05-14T08:39:37.018552Z","iopub.status.idle":"2023-05-14T08:39:37.03272Z","shell.execute_reply.started":"2023-05-14T08:39:37.018509Z","shell.execute_reply":"2023-05-14T08:39:37.031708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def map_new_to_old_style(sequence):\n    types = []\n    landmark_indexes = []\n    for column in list(sequence.columns)[1:544]:\n        parts = column.split(\"_\")\n        if len(parts) == 4:\n            types.append(parts[1] + \"_\" + parts[2])\n        else:\n            types.append(parts[1])\n\n        landmark_indexes.append(int(parts[-1]))\n\n    data = {\n        \"frame\": [],\n        \"type\": [],\n        \"landmark_index\": [],\n        \"x\": [],\n        \"y\": [],\n        \"z\": []\n    }\n\n    for index, row in sequence.iterrows():\n        data[\"frame\"] += [int(row.frame)]*543\n        data[\"type\"] += types\n        data[\"landmark_index\"] += landmark_indexes\n\n        for _type, landmark_index in zip(types, landmark_indexes):\n            data[\"x\"].append(row[f\"x_{_type}_{landmark_index}\"])\n            data[\"y\"].append(row[f\"y_{_type}_{landmark_index}\"])\n            data[\"z\"].append(row[f\"z_{_type}_{landmark_index}\"])\n\n    return pd.DataFrame.from_dict(data)\n\n# assign desired colors to landmarks\ndef assign_color(row):\n    if row == 'face':\n        return 'red'\n    elif 'hand' in row:\n        return 'dodgerblue'\n    else:\n        return 'green'\n\n# specifies the plotting order\ndef assign_order(row):\n    if row.type == 'face':\n        return row.landmark_index + 101\n    elif row.type == 'pose':\n        return row.landmark_index + 30\n    elif row.type == 'left_hand':\n        return row.landmark_index + 80\n    else:\n        return row.landmark_index\n\ndef visualise2d_landmarks(parquet_df, title=\"\"):\n    connections = [  \n        [0, 1, 2, 3, 4,],\n        [0, 5, 6, 7, 8],\n        [0, 9, 10, 11, 12],\n        [0, 13, 14, 15, 16],\n        [0, 17, 18, 19, 20],\n\n        \n        [38, 36, 35, 34, 30, 31, 32, 33, 37],\n        [40, 39],\n        [52, 46, 50, 48, 46, 44, 42, 41, 43, 45, 47, 49, 45, 51],\n        [42, 54, 56, 58, 60, 62, 58],\n        [41, 53, 55, 57, 59, 61, 57],\n        [54, 53],\n\n        \n        [80, 81, 82, 83, 84, ],\n        [80, 85, 86, 87, 88],\n        [80, 89, 90, 91, 92],\n        [80, 93, 94, 95, 96],\n        [80, 97, 98, 99, 100], ]\n\n    parquet_df = map_new_to_old_style(parquet_df)\n    frames = sorted(set(parquet_df.frame))\n    first_frame = min(frames)\n    parquet_df['color'] = parquet_df.type.apply(lambda row: assign_color(row))\n    parquet_df['plot_order'] = parquet_df.apply(lambda row: assign_order(row), axis=1)\n    first_frame_df = parquet_df[parquet_df.frame == first_frame].copy()\n    first_frame_df = first_frame_df.sort_values([\"plot_order\"]).set_index('plot_order')\n\n\n    frames_l = []\n    for frame in frames:\n        filtered_df = parquet_df[parquet_df.frame == frame].copy()\n        filtered_df = filtered_df.sort_values([\"plot_order\"]).set_index(\"plot_order\")\n        traces = [go.Scatter(\n            x=filtered_df['x'],\n            y=filtered_df['y'],\n            mode='markers',\n            marker=dict(\n                color=filtered_df.color,\n                size=9))]\n\n        for i, seg in enumerate(connections):\n            trace = go.Scatter(\n                    x=filtered_df.loc[seg]['x'],\n                    y=filtered_df.loc[seg]['y'],\n                    mode='lines',\n            )\n            traces.append(trace)\n        frame_data = go.Frame(data=traces, traces = [i for i in range(17)])\n        frames_l.append(frame_data)\n\n    traces = [go.Scatter(\n        x=first_frame_df['x'],\n        y=first_frame_df['y'],\n        mode='markers',\n        marker=dict(\n            color=first_frame_df.color,\n            size=9\n        )\n    )]\n    for i, seg in enumerate(connections):\n        trace = go.Scatter(\n            x=first_frame_df.loc[seg]['x'],\n            y=first_frame_df.loc[seg]['y'],\n            mode='lines',\n            line=dict(\n                color='black',\n                width=2\n            )\n        )\n        traces.append(trace)\n    fig = go.Figure(\n        data=traces,\n        frames=frames_l\n    )\n\n\n    fig.update_layout(\n        width=500,\n        height=800,\n        scene={\n            'aspectmode': 'data',\n        },\n        updatemenus=[\n            {\n                \"buttons\": [\n                    {\n                        \"args\": [None, {\"frame\": {\"duration\": 100,\n                                                  \"redraw\": True},\n                                        \"fromcurrent\": True,\n                                        \"transition\": {\"duration\": 0}}],\n                        \"label\": \"▶\",\n                        \"method\": \"animate\",\n                    },\n\n                ],\n                \"direction\": \"left\",\n                \"pad\": {\"r\": 100, \"t\": 100},\n                \"font\": {\"size\":30},\n                \"type\": \"buttons\",\n                \"x\": 0.1,\n                \"y\": 0,\n            }\n        ],\n    )\n    camera = dict(\n        up=dict(x=0, y=-1, z=0),\n        eye=dict(x=0, y=0, z=2.5)\n    )\n    fig.update_layout(title_text=title, title_x=0.5)\n    fig.update_layout(scene_camera=camera, showlegend=False)\n    fig.update_layout(xaxis = dict(visible=False),\n            yaxis = dict(visible=False),\n    )\n    fig.update_yaxes(autorange=\"reversed\")\n\n    fig.show()\n\n\ndef get_phrase(df, file_id, sequence_id):\n    return df[\n        np.logical_and(\n            df.file_id == file_id, \n            df.sequence_id == sequence_id\n        )\n    ].phrase.iloc[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:37.037095Z","iopub.execute_input":"2023-05-14T08:39:37.037419Z","iopub.status.idle":"2023-05-14T08:39:37.06249Z","shell.execute_reply.started":"2023-05-14T08:39:37.037392Z","shell.execute_reply":"2023-05-14T08:39:37.061661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualise2d_landmarks(read_one_parquet_data, f\"Phrase: \")","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:39:37.063801Z","iopub.execute_input":"2023-05-14T08:39:37.064416Z","iopub.status.idle":"2023-05-14T08:41:36.713367Z","shell.execute_reply.started":"2023-05-14T08:39:37.064384Z","shell.execute_reply":"2023-05-14T08:41:36.711866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Character To Prediction Index Data","metadata":{}},{"cell_type":"code","source":"with open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as user_file:\n  file_contents = user_file.read()\n  \nprint(file_contents)\n\nread_charecter_json = json.loads(file_contents)","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.715605Z","iopub.execute_input":"2023-05-14T08:41:36.715983Z","iopub.status.idle":"2023-05-14T08:41:36.744086Z","shell.execute_reply.started":"2023-05-14T08:41:36.715951Z","shell.execute_reply":"2023-05-14T08:41:36.742688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_charecter_json","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.74506Z","iopub.status.idle":"2023-05-14T08:41:36.745495Z","shell.execute_reply.started":"2023-05-14T08:41:36.745283Z","shell.execute_reply":"2023-05-14T08:41:36.745303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(read_charecter_json['0'])","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.747688Z","iopub.status.idle":"2023-05-14T08:41:36.74811Z","shell.execute_reply.started":"2023-05-14T08:41:36.747917Z","shell.execute_reply":"2023-05-14T08:41:36.747936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Data","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/asl-fingerspelling/train.csv\")\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.749747Z","iopub.status.idle":"2023-05-14T08:41:36.75015Z","shell.execute_reply.started":"2023-05-14T08:41:36.749962Z","shell.execute_reply":"2023-05-14T08:41:36.749981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_seq_id = 1816796431\ntrain_field_id = 5414471","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.752291Z","iopub.status.idle":"2023-05-14T08:41:36.752739Z","shell.execute_reply.started":"2023-05-14T08:41:36.752516Z","shell.execute_reply":"2023-05-14T08:41:36.752536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_perq_data = pd.read_parquet(f'/kaggle/input/asl-fingerspelling/train_landmarks/{train_field_id}.parquet')\ntrain_perq_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.754396Z","iopub.status.idle":"2023-05-14T08:41:36.754831Z","shell.execute_reply.started":"2023-05-14T08:41:36.754632Z","shell.execute_reply":"2023-05-14T08:41:36.754653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_perq_data = train_perq_data.dropna().reset_index(drop=True)\n# train_perq_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.756533Z","iopub.status.idle":"2023-05-14T08:41:36.757556Z","shell.execute_reply.started":"2023-05-14T08:41:36.757332Z","shell.execute_reply":"2023-05-14T08:41:36.757355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_perq_data.index == 1816796431","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.759264Z","iopub.status.idle":"2023-05-14T08:41:36.760072Z","shell.execute_reply.started":"2023-05-14T08:41:36.759857Z","shell.execute_reply":"2023-05-14T08:41:36.75988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_perq_data = train_perq_data[train_perq_data.index == train_seq_id]\ntrain_perq_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.761019Z","iopub.status.idle":"2023-05-14T08:41:36.762324Z","shell.execute_reply.started":"2023-05-14T08:41:36.762113Z","shell.execute_reply":"2023-05-14T08:41:36.762136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_phrase = get_phrase(train_data, train_field_id, train_seq_id)\nvisualise2d_landmarks(train_perq_data, f\"Phrase: {sequence_phrase}\")","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.763447Z","iopub.status.idle":"2023-05-14T08:41:36.763863Z","shell.execute_reply.started":"2023-05-14T08:41:36.763668Z","shell.execute_reply":"2023-05-14T08:41:36.763687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_perq_data['phrase'] = sequence_phrase\ntrain_perq_data","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.765549Z","iopub.status.idle":"2023-05-14T08:41:36.765973Z","shell.execute_reply.started":"2023-05-14T08:41:36.765789Z","shell.execute_reply":"2023-05-14T08:41:36.765808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nfrom keras.models import Sequential\nfrom keras.utils import np_utils\nfrom keras.layers import Dense,Activation,Flatten,Dropout,Conv2D,MaxPooling2D\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.768154Z","iopub.status.idle":"2023-05-14T08:41:36.768539Z","shell.execute_reply.started":"2023-05-14T08:41:36.768358Z","shell.execute_reply":"2023-05-14T08:41:36.768376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32 , (3,3) , padding='same' , input_shape=xtrain.shape[1:]))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(32 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.1))\n\nmodel.add(Conv2D(64 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(64 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n#model.add(Dropout(0.1))\n\nmodel.add(Conv2D(128 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(128 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(128 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n#model.add(Dropout(0.1))\n\nmodel.add(Conv2D(256 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(256 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(256 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n#model.add(Dropout(0.1))\n\nmodel.add(Conv2D(512 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(512 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(Conv2D(512 , (3,3) , padding='same'))\nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n#model.add(Dropout(0.1))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(15,activation='sigmoid'))","metadata":{"execution":{"iopub.status.busy":"2023-05-14T08:41:36.770325Z","iopub.status.idle":"2023-05-14T08:41:36.770735Z","shell.execute_reply.started":"2023-05-14T08:41:36.770523Z","shell.execute_reply":"2023-05-14T08:41:36.770542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}