{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Set up","metadata":{}},{"cell_type":"code","source":"from json import load\nfrom typing import List\n\nimport numpy as np\nfrom pandas import DataFrame, Series, read_csv, read_parquet\nfrom matplotlib.pyplot import show","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-25T04:54:47.402287Z","iopub.execute_input":"2023-06-25T04:54:47.402996Z","iopub.status.idle":"2023-06-25T04:54:47.410554Z","shell.execute_reply.started":"2023-06-25T04:54:47.402934Z","shell.execute_reply":"2023-06-25T04:54:47.408899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = \"/kaggle/input/asl-fingerspelling\"","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.420239Z","iopub.execute_input":"2023-06-25T04:54:47.421284Z","iopub.status.idle":"2023-06-25T04:54:47.427972Z","shell.execute_reply.started":"2023-06-25T04:54:47.421228Z","shell.execute_reply":"2023-06-25T04:54:47.426409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"markdown","source":"## Explore training data","metadata":{}},{"cell_type":"code","source":"train = read_csv(f\"{BASE_DIR}/train.csv\")\ntrain.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.430634Z","iopub.execute_input":"2023-06-25T04:54:47.431173Z","iopub.status.idle":"2023-06-25T04:54:47.596648Z","shell.execute_reply.started":"2023-06-25T04:54:47.431125Z","shell.execute_reply":"2023-06-25T04:54:47.594458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(\"path\", axis=1).set_index('sequence_id')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.600572Z","iopub.execute_input":"2023-06-25T04:54:47.601516Z","iopub.status.idle":"2023-06-25T04:54:47.621419Z","shell.execute_reply.started":"2023-06-25T04:54:47.601465Z","shell.execute_reply":"2023-06-25T04:54:47.619798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_samples = len(train)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.623574Z","iopub.execute_input":"2023-06-25T04:54:47.624204Z","iopub.status.idle":"2023-06-25T04:54:47.629316Z","shell.execute_reply.started":"2023-06-25T04:54:47.624157Z","shell.execute_reply":"2023-06-25T04:54:47.628159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\n    f\"Training set has {n_samples} samples \"\n    f\"in {train.file_id.nunique()} files \"\n    f\"from {train.participant_id.nunique()} participants \"\n    f\"with {train.phrase.nunique()} unique phrases\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.632702Z","iopub.execute_input":"2023-06-25T04:54:47.633923Z","iopub.status.idle":"2023-06-25T04:54:47.670163Z","shell.execute_reply.started":"2023-06-25T04:54:47.633876Z","shell.execute_reply":"2023-06-25T04:54:47.668777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top phrases","metadata":{}},{"cell_type":"code","source":"n = 30\ntop_phrases = train[\"phrase\"].value_counts().head(n)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.671924Z","iopub.execute_input":"2023-06-25T04:54:47.672429Z","iopub.status.idle":"2023-06-25T04:54:47.725541Z","shell.execute_reply.started":"2023-06-25T04:54:47.672385Z","shell.execute_reply":"2023-06-25T04:54:47.724449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_phrases.sort_values(ascending=True).plot(kind=\"barh\", figsize=(10,10), title=f\"Top {n} phrases\", xlabel=\"Number of samples\", colormap=\"jet\")\nshow()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:47.727508Z","iopub.execute_input":"2023-06-25T04:54:47.727978Z","iopub.status.idle":"2023-06-25T04:54:48.381284Z","shell.execute_reply.started":"2023-06-25T04:54:47.727938Z","shell.execute_reply":"2023-06-25T04:54:48.3802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore supplemental data ","metadata":{}},{"cell_type":"code","source":"supplemental = read_csv(f\"{BASE_DIR}/supplemental_metadata.csv\").drop(\"path\", axis=1).set_index('sequence_id')\nsupplemental.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.38264Z","iopub.execute_input":"2023-06-25T04:54:48.383019Z","iopub.status.idle":"2023-06-25T04:54:48.527288Z","shell.execute_reply.started":"2023-06-25T04:54:48.38298Z","shell.execute_reply":"2023-06-25T04:54:48.526409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\n    f\"Supplemental data has {len(supplemental)} samples \"\n    f\"in {train.file_id.nunique()} files \"\n    f\"from {train.participant_id.nunique()} participants \"\n    f\"with {train.phrase.nunique()} unique phrases\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.528582Z","iopub.execute_input":"2023-06-25T04:54:48.529258Z","iopub.status.idle":"2023-06-25T04:54:48.558893Z","shell.execute_reply.started":"2023-06-25T04:54:48.529226Z","shell.execute_reply":"2023-06-25T04:54:48.557684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Character to ordinal mapping","metadata":{}},{"cell_type":"code","source":"# Read Character to Ordinal Encoding Mapping\nwith open(f\"{BASE_DIR}/character_to_prediction_index.json\", \"r\") as file:\n    character_to_ordinal = load(file)\n    \n# Ordinal to Character Mapping\nordinal_to_character = {j:i for i,j in character_to_ordinal.items()}\n    \n# Character to Ordinal Encoding Mapping   \nSeries(character_to_ordinal).to_frame('Ordinal Encoding')","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.560242Z","iopub.execute_input":"2023-06-25T04:54:48.560599Z","iopub.status.idle":"2023-06-25T04:54:48.579856Z","shell.execute_reply.started":"2023-06-25T04:54:48.560554Z","shell.execute_reply":"2023-06-25T04:54:48.578417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature selection","metadata":{}},{"cell_type":"code","source":"HAND_INDEXES = range(0, 21)\nLIPS_LANDMARK_INDEXES = [\n    61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 146, 91, 181, 84, 17, 314, 405, 321, 375,\n    78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,\n]\n\nselected_features_indexes = {\n    \"right_hand\": HAND_INDEXES,\n    \"left_hand\": HAND_INDEXES,\n    \"face\": LIPS_LANDMARK_INDEXES,\n}","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.584967Z","iopub.execute_input":"2023-06-25T04:54:48.585378Z","iopub.status.idle":"2023-06-25T04:54:48.592868Z","shell.execute_reply.started":"2023-06-25T04:54:48.585324Z","shell.execute_reply":"2023-06-25T04:54:48.591678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select only the x and y diemnsions of the lips and hands\nselected_columns = [\n    f\"{dimension}_{landmark_type}_{index}\" \n    for dimension in [\"x\", \"y\"]\n    for landmark_type, index_range in selected_features_indexes.items() \n    for index in index_range \n]\nSeries(selected_columns).to_frame(\"Selected Features\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.594722Z","iopub.execute_input":"2023-06-25T04:54:48.595157Z","iopub.status.idle":"2023-06-25T04:54:48.614105Z","shell.execute_reply.started":"2023-06-25T04:54:48.595119Z","shell.execute_reply":"2023-06-25T04:54:48.612757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path: str) -> DataFrame:\n    return read_parquet(pq_path, columns=selected_columns)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.615959Z","iopub.execute_input":"2023-06-25T04:54:48.616489Z","iopub.status.idle":"2023-06-25T04:54:48.623793Z","shell.execute_reply.started":"2023-06-25T04:54:48.616457Z","shell.execute_reply":"2023-06-25T04:54:48.622786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sample file","metadata":{}},{"cell_type":"code","source":"sample_data = load_relevant_data_subset(f\"{BASE_DIR}/train_landmarks/5414471.parquet\")\nsample_data.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:48.625295Z","iopub.execute_input":"2023-06-25T04:54:48.626322Z","iopub.status.idle":"2023-06-25T04:54:51.016867Z","shell.execute_reply.started":"2023-06-25T04:54:48.626288Z","shell.execute_reply":"2023-06-25T04:54:51.015671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_data.describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:51.01828Z","iopub.execute_input":"2023-06-25T04:54:51.018662Z","iopub.status.idle":"2023-06-25T04:54:52.547671Z","shell.execute_reply.started":"2023-06-25T04:54:51.018632Z","shell.execute_reply":"2023-06-25T04:54:52.546242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sample sequence","metadata":{}},{"cell_type":"code","source":"sequence_id = 1816796431","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.549175Z","iopub.execute_input":"2023-06-25T04:54:52.549886Z","iopub.status.idle":"2023-06-25T04:54:52.555802Z","shell.execute_reply.started":"2023-06-25T04:54:52.549846Z","shell.execute_reply":"2023-06-25T04:54:52.554419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select all frames of sequence \nsequence_data = sample_data.loc[sequence_id]\nsequence_data","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.557368Z","iopub.execute_input":"2023-06-25T04:54:52.557808Z","iopub.status.idle":"2023-06-25T04:54:52.611724Z","shell.execute_reply.started":"2023-06-25T04:54:52.557774Z","shell.execute_reply":"2023-06-25T04:54:52.610382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence_data = sequence_data.fillna(method='ffill')\nsequence_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.613299Z","iopub.execute_input":"2023-06-25T04:54:52.613719Z","iopub.status.idle":"2023-06-25T04:54:52.647922Z","shell.execute_reply.started":"2023-06-25T04:54:52.613687Z","shell.execute_reply":"2023-06-25T04:54:52.646918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_landmark_data(data: DataFrame, landmark: str, index_range: List[int]) -> DataFrame:\n    return data[[f\"{d}_{landmark}_{i}\" for d in [\"x\", \"y\"] for i in index_range]].dropna(how=\"all\")","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.649405Z","iopub.execute_input":"2023-06-25T04:54:52.649974Z","iopub.status.idle":"2023-06-25T04:54:52.656427Z","shell.execute_reply.started":"2023-06-25T04:54:52.649944Z","shell.execute_reply":"2023-06-25T04:54:52.654892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"right_hand_data = get_landmark_data(sequence_data, \"right_hand\", HAND_INDEXES)\nright_hand_data","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.658095Z","iopub.execute_input":"2023-06-25T04:54:52.658541Z","iopub.status.idle":"2023-06-25T04:54:52.704669Z","shell.execute_reply.started":"2023-06-25T04:54:52.65849Z","shell.execute_reply":"2023-06-25T04:54:52.703172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"left_hand_data = get_landmark_data(sequence_data, \"left_hand\", HAND_INDEXES)\nleft_hand_data","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.706745Z","iopub.execute_input":"2023-06-25T04:54:52.707159Z","iopub.status.idle":"2023-06-25T04:54:52.731755Z","shell.execute_reply.started":"2023-06-25T04:54:52.707123Z","shell.execute_reply":"2023-06-25T04:54:52.730201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lips_data = get_landmark_data(sequence_data, \"face\", LIPS_LANDMARK_INDEXES)\nlips_data","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.733264Z","iopub.execute_input":"2023-06-25T04:54:52.733793Z","iopub.status.idle":"2023-06-25T04:54:52.776937Z","shell.execute_reply.started":"2023-06-25T04:54:52.733735Z","shell.execute_reply":"2023-06-25T04:54:52.775601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get corresponding phrase\nphrase = train.loc[sequence_id, \"phrase\"]\nprint(phrase)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.779767Z","iopub.execute_input":"2023-06-25T04:54:52.780301Z","iopub.status.idle":"2023-06-25T04:54:52.790303Z","shell.execute_reply.started":"2023-06-25T04:54:52.780243Z","shell.execute_reply":"2023-06-25T04:54:52.788784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeled_sequence = DataFrame({\"sequence_id\": sequence_id, \"character\": list(phrase)})\nlabeled_sequence[\"label\"] = labeled_sequence[\"character\"].map(character_to_ordinal).astype(np.int8)\nlabeled_sequence = labeled_sequence.set_index(\"sequence_id\")\nlabeled_sequence","metadata":{"execution":{"iopub.status.busy":"2023-06-25T04:54:52.791993Z","iopub.execute_input":"2023-06-25T04:54:52.792375Z","iopub.status.idle":"2023-06-25T04:54:52.816649Z","shell.execute_reply.started":"2023-06-25T04:54:52.79233Z","shell.execute_reply":"2023-06-25T04:54:52.815074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference notebooks\n- [ASLFR - Transformer Training + Inference 🤟](https://www.kaggle.com/code/markwijkhuizen/aslfr-transformer-training-inference)\n- [ASLFR EDA & preprocessing](https://www.kaggle.com/code/hebasaleh00/aslfr-eda-preprocessing)\n- [👌American Sign Language Recognition - EDA](https://www.kaggle.com/code/mahakpreetkaurvirdi/american-sign-language-recognition-eda)\n\nRemember to upvote if you find any of them useful","metadata":{}}]}