{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Edge cases\n\nSubmission failures are occurring for many participants, despite local inference working fine.\n\nAfter TFLite runtime versions were found to probably not be the issue ([TFLite Runtime version incompatibilities or bugs](https://www.kaggle.com/competitions/asl-fingerspelling/discussion/414682)), the reason is probably edge cases in the test set.\n\nThis notebook handles two of those edge cases in the `Preprocessing` layer:\n\n- Empty samples\n- Samples where no hand keypoints are present\n\nThe latter edge case may explain why existing notebooks like [this one](https://www.kaggle.com/code/irohith/aslfr-transformer) passed submission: they also use lip keypoints, and so the preprocessing code will not generate empty samples.\n\nHowever, this notebook still does not pass the scoring pipeline *despite working on all training and supplemental files* so there may be additional edge cases not accounted for (or the submission errors are because of another reason).\n\nThe bottom of this notebook also provides example code to run the model on the entire training and supplemental landmarks sets, to show that local inference with the TFLite Runtime works without errors.\n\n## How to use\n\nYou cannot use both TFLite Runtime version 2.14 and TensorFlow 2.12 in the same script. To test out local inference, turn on Persistence (Files only) in your notebook, run all of the code until the inference section, do a Factory reset (under the Run menu), and then run *only* the inference section.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"! python --version\n\nimport tensorflow as tf\ntf.__version__","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:47.815639Z","iopub.execute_input":"2023-06-03T10:29:47.816104Z","iopub.status.idle":"2023-06-03T10:29:48.934083Z","shell.execute_reply.started":"2023-06-03T10:29:47.816064Z","shell.execute_reply":"2023-06-03T10:29:48.932722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = tf.zeros((10, 126))\nnon_empty_frames_idxs = tf.where(tf.math.reduce_sum(x, axis=1) > 0)\nempty_sample = tf.shape(non_empty_frames_idxs)[0] == 0\nnon_empty_frames_idxs = tf.cond(empty_sample, lambda: tf.ones((1, 1), dtype=tf.int32), lambda: non_empty_frame_idxs)\nprint(non_empty_frames_idxs)\nx = tf.gather(x, non_empty_frames_idxs, axis=0, name='gather_non_empty_frames_idxs')","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:48.936752Z","iopub.execute_input":"2023-06-03T10:29:48.937155Z","iopub.status.idle":"2023-06-03T10:29:48.951657Z","shell.execute_reply.started":"2023-06-03T10:29:48.937115Z","shell.execute_reply":"2023-06-03T10:29:48.950157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\nwith open(\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\") as f:\n    character_map = json.load(f)\n\nnumber_of_characters = len(character_map)\nnumber_of_characters","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:48.952652Z","iopub.execute_input":"2023-06-03T10:29:48.952955Z","iopub.status.idle":"2023-06-03T10:29:48.967125Z","shell.execute_reply.started":"2023-06-03T10:29:48.952929Z","shell.execute_reply":"2023-06-03T10:29:48.965956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['x_right_hand_0','y_right_hand_0','z_right_hand_0',\n            'x_right_hand_1','y_right_hand_1','z_right_hand_1',\n            'x_right_hand_2','y_right_hand_2','z_right_hand_2',\n            'x_right_hand_3','y_right_hand_3','z_right_hand_3',\n            'x_right_hand_4','y_right_hand_4','z_right_hand_4',\n            'x_right_hand_5','y_right_hand_5','z_right_hand_5',\n            'x_right_hand_6','y_right_hand_6','z_right_hand_6',\n            'x_right_hand_7','y_right_hand_7','z_right_hand_7',\n            'x_right_hand_8','y_right_hand_8','z_right_hand_8',\n            'x_right_hand_9','y_right_hand_9','z_right_hand_9',\n            'x_right_hand_10','y_right_hand_10','z_right_hand_10',\n            'x_right_hand_11','y_right_hand_11','z_right_hand_11',\n            'x_right_hand_12','y_right_hand_12','z_right_hand_12',\n            'x_right_hand_13','y_right_hand_13','z_right_hand_13',\n            'x_right_hand_14','y_right_hand_14','z_right_hand_14',\n            'x_right_hand_15','y_right_hand_15','z_right_hand_15',\n            'x_right_hand_16','y_right_hand_16','z_right_hand_16',\n            'x_right_hand_17','y_right_hand_17','z_right_hand_17',\n            'x_right_hand_18','y_right_hand_18','z_right_hand_18',\n            'x_right_hand_19','y_right_hand_19','z_right_hand_19',\n            'x_right_hand_20','y_right_hand_20','z_right_hand_20',\n            'x_left_hand_0','y_left_hand_0','z_left_hand_0',\n            'x_left_hand_1','y_left_hand_1','z_left_hand_1',\n            'x_left_hand_2','y_left_hand_2','z_left_hand_2',\n            'x_left_hand_3','y_left_hand_3','z_left_hand_3',\n            'x_left_hand_4','y_left_hand_4','z_left_hand_4',\n            'x_left_hand_5','y_left_hand_5','z_left_hand_5',\n            'x_left_hand_6','y_left_hand_6','z_left_hand_6',\n            'x_left_hand_7','y_left_hand_7','z_left_hand_7',\n            'x_left_hand_8','y_left_hand_8','z_left_hand_8',\n            'x_left_hand_9','y_left_hand_9','z_left_hand_9',\n            'x_left_hand_10','y_left_hand_10','z_left_hand_10',\n            'x_left_hand_11','y_left_hand_11','z_left_hand_11',\n            'x_left_hand_12','y_left_hand_12','z_left_hand_12',\n            'x_left_hand_13','y_left_hand_13','z_left_hand_13',\n            'x_left_hand_14','y_left_hand_14','z_left_hand_14',\n            'x_left_hand_15','y_left_hand_15','z_left_hand_15',\n            'x_left_hand_16','y_left_hand_16','z_left_hand_16',\n            'x_left_hand_17','y_left_hand_17','z_left_hand_17',\n            'x_left_hand_18','y_left_hand_18','z_left_hand_18',\n            'x_left_hand_19','y_left_hand_19','z_left_hand_19',\n            'x_left_hand_20','y_left_hand_20','z_left_hand_20'\n]\nnumber_of_features = len(features)\nnumber_of_features","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:48.970079Z","iopub.execute_input":"2023-06-03T10:29:48.970477Z","iopub.status.idle":"2023-06-03T10:29:48.9895Z","shell.execute_reply.started":"2023-06-03T10:29:48.970444Z","shell.execute_reply":"2023-06-03T10:29:48.988123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('inference_args.json', 'w') as f:\n    json.dump({'selected_columns': features}, f)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:48.991174Z","iopub.execute_input":"2023-06-03T10:29:48.991645Z","iopub.status.idle":"2023-06-03T10:29:49.002417Z","shell.execute_reply.started":"2023-06-03T10:29:48.991611Z","shell.execute_reply":"2023-06-03T10:29:49.001423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Preprocessing(tf.keras.layers.Layer):\n    def __init__(self):\n        super(Preprocessing, self).__init__()\n        \n    def __call__(self, x):\n        # Empty sample check.\n        # Some samples are empty in the supplemental set, and maybe also in the test set.\n        # Replace them with some random frames.\n        empty_sample = tf.shape(x)[0] == 0\n        x = tf.cond(empty_sample, lambda: tf.random.uniform((10, number_of_features)), lambda: x)\n        \n        # NaN removal.\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        \n        # Missing frame removal.\n        # Some samples will have missing hands for all frames.\n        # Make sure to avoid doing a `tf.gather` operation for those.\n        non_empty_frames_idxs = tf.where(tf.math.reduce_sum(x, axis=1) > 0)\n        empty_sample = tf.shape(non_empty_frames_idxs)[0] == 0\n        non_empty_frames_idxs = tf.cond(empty_sample, lambda: tf.ones((1, 1), dtype=tf.int64), lambda: non_empty_frames_idxs)\n        x = tf.gather(x, non_empty_frames_idxs[0], axis=0, name='gather_non_empty_frames_idxs')\n        \n        return x\n    \nclass Model(tf.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n\n        self.preprocessing = Preprocessing()\n        self.dummy = tf.keras.layers.Dense(number_of_characters)\n    \n    @tf.function(input_signature=[tf.TensorSpec(shape=[None, number_of_features], dtype=tf.float32, name='inputs')])\n    def __call__(self, inputs):\n        x = self.preprocessing(inputs)\n        x = x[None]\n        x = self.dummy(x)\n        x = x[0]\n        return {'outputs': x}","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:49.003811Z","iopub.execute_input":"2023-06-03T10:29:49.004509Z","iopub.status.idle":"2023-06-03T10:29:49.019733Z","shell.execute_reply.started":"2023-06-03T10:29:49.004477Z","shell.execute_reply":"2023-06-03T10:29:49.018507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model()\nmodel(tf.random.uniform((10, number_of_features)))\n\nconverter = tf.lite.TFLiteConverter.from_keras_model(model)\ntflite_model = converter.convert()\n\nwith open('model.tflite', 'wb') as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:49.021161Z","iopub.execute_input":"2023-06-03T10:29:49.02149Z","iopub.status.idle":"2023-06-03T10:29:50.283524Z","shell.execute_reply.started":"2023-06-03T10:29:49.021463Z","shell.execute_reply":"2023-06-03T10:29:50.282021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip model.tflite inference_args.json","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:29:50.285238Z","iopub.execute_input":"2023-06-03T10:29:50.28566Z","iopub.status.idle":"2023-06-03T10:29:51.400014Z","shell.execute_reply.started":"2023-06-03T10:29:50.28562Z","shell.execute_reply":"2023-06-03T10:29:51.398422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# Set this to `True` to run inference on all of the provided data.\n# Read the \"how to use\" section at the top of this notebook before trying.\nenable_inference = False","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:32:22.861185Z","iopub.execute_input":"2023-06-03T10:32:22.862059Z","iopub.status.idle":"2023-06-03T10:32:22.899501Z","shell.execute_reply.started":"2023-06-03T10:32:22.862019Z","shell.execute_reply":"2023-06-03T10:32:22.898561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if enable_inference:\n    import pandas as pd\n    import json\n    from tqdm import tqdm\n    import numpy as np\n    !pip install tflite-runtime-nightly==2.14.0.dev20230510\n\n    with open(\"inference_args.json\") as f:\n        selected_columns = json.load(f)[\"selected_columns\"]\n\n    def load_relevant_data_subset(pq_path):\n        return pd.read_parquet(pq_path, columns=selected_columns)\n\n    model_path = \"model.tflite\"\n\n    import tflite_runtime.interpreter as tflite\n    interpreter = tflite.Interpreter(model_path)\n\n    REQUIRED_SIGNATURE = \"serving_default\"\n    REQUIRED_OUTPUT = \"outputs\"\n\n    with open (\"/kaggle/input/asl-fingerspelling/character_to_prediction_index.json\", \"r\") as f:\n        character_map = json.load(f)\n    rev_character_map = {j:i for i,j in character_map.items()}\n\n    found_signatures = list(interpreter.get_signature_list().keys())\n\n    if REQUIRED_SIGNATURE not in found_signatures:\n        raise KernelEvalException('Required input signature not found.')\n\n    prediction_fn = interpreter.get_signature_runner(REQUIRED_SIGNATURE)\n\n    # Training set.\n    data_path = None\n    df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\n    for row_index, row in tqdm(df.iterrows()):\n        sequence_id = row.sequence_id\n\n        if data_path is None:\n            data_path = '/kaggle/input/asl-fingerspelling/' + row.path\n            frames = load_relevant_data_subset(data_path)\n\n        # Avoid loading same file twice in a row.\n        if data_path != '/kaggle/input/asl-fingerspelling/' + row.path:\n            frames = load_relevant_data_subset('/kaggle/input/asl-fingerspelling/' + row.path)\n            data_path = '/kaggle/input/asl-fingerspelling/' + row.path\n\n        inputs = frames.loc[frames.index == sequence_id].values\n        output = prediction_fn(inputs=inputs)\n        prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])\n\n    # Supplemental set.\n    data_path = None\n    df = pd.read_csv('/kaggle/input/asl-fingerspelling/supplemental_metadata.csv')\n    for row_index, row in tqdm(df.iterrows()):\n        sequence_id = row.sequence_id\n\n        if data_path is None:\n            data_path = '/kaggle/input/asl-fingerspelling/' + row.path\n            frames = load_relevant_data_subset(data_path)\n\n        # Avoid loading same file twice in a row.\n        if data_path != '/kaggle/input/asl-fingerspelling/' + row.path:\n            frames = load_relevant_data_subset('/kaggle/input/asl-fingerspelling/' + row.path)\n            data_path = '/kaggle/input/asl-fingerspelling/' + row.path\n\n        inputs = frames.loc[frames.index == sequence_id].values\n        output = prediction_fn(inputs=inputs)\n        prediction_str = \"\".join([rev_character_map.get(s, \"\") for s in np.argmax(output[REQUIRED_OUTPUT], axis=1)])","metadata":{"execution":{"iopub.status.busy":"2023-06-03T10:32:23.080619Z","iopub.execute_input":"2023-06-03T10:32:23.081773Z","iopub.status.idle":"2023-06-03T10:35:53.214788Z","shell.execute_reply.started":"2023-06-03T10:32:23.081729Z","shell.execute_reply":"2023-06-03T10:35:53.213473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}