{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\n\nfrom joblib import Parallel, delayed\nimport multiprocessing as mp\nfrom multiprocessing import cpu_count\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold, KFold\n\nimport tensorflow as tf\ncpu_count()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-08T08:20:34.300071Z","iopub.execute_input":"2023-06-08T08:20:34.301675Z","iopub.status.idle":"2023-06-08T08:20:45.678228Z","shell.execute_reply.started":"2023-06-08T08:20:34.3016Z","shell.execute_reply":"2023-06-08T08:20:45.676782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = pd.read_csv('/kaggle/input/asl-signs/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:20:45.681183Z","iopub.execute_input":"2023-06-08T08:20:45.682068Z","iopub.status.idle":"2023-06-08T08:20:45.687625Z","shell.execute_reply.started":"2023-06-08T08:20:45.682014Z","shell.execute_reply":"2023-06-08T08:20:45.686149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/asl-fingerspelling/train.csv')\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:51:04.127911Z","iopub.execute_input":"2023-06-08T08:51:04.128395Z","iopub.status.idle":"2023-06-08T08:51:04.327319Z","shell.execute_reply.started":"2023-06-08T08:51:04.128349Z","shell.execute_reply":"2023-06-08T08:51:04.325961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.read_parquet('/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet')\n# temp.frame.groupby('sequence_id').agg('max').max()\n# temp.index.value_counts()\ntemp","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:23:07.257013Z","iopub.execute_input":"2023-06-08T08:23:07.257467Z","iopub.status.idle":"2023-06-08T08:23:10.792968Z","shell.execute_reply.started":"2023-06-08T08:23:07.257424Z","shell.execute_reply":"2023-06-08T08:23:10.791709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp.loc[temp.index==1975433633]","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:49:14.034541Z","iopub.execute_input":"2023-06-08T08:49:14.035055Z","iopub.status.idle":"2023-06-08T08:49:14.078119Z","shell.execute_reply.started":"2023-06-08T08:49:14.035011Z","shell.execute_reply":"2023-06-08T08:49:14.076631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.read_parquet('/kaggle/input/asl-signs/train_landmark_files/26734/1000035562.parquet')#[:543].type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:22:45.366796Z","iopub.execute_input":"2023-06-08T08:22:45.367284Z","iopub.status.idle":"2023-06-08T08:22:45.374165Z","shell.execute_reply.started":"2023-06-08T08:22:45.367242Z","shell.execute_reply":"2023-06-08T08:22:45.372163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.read_parquet('/kaggle/input/asl-signs/train_landmark_files/26734/1000035562.parquet')[543:543*2].type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:20:46.089719Z","iopub.execute_input":"2023-06-08T08:20:46.091189Z","iopub.status.idle":"2023-06-08T08:20:46.097292Z","shell.execute_reply.started":"2023-06-08T08:20:46.091134Z","shell.execute_reply":"2023-06-08T08:20:46.095757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:20:46.098809Z","iopub.execute_input":"2023-06-08T08:20:46.099171Z","iopub.status.idle":"2023-06-08T08:20:46.113439Z","shell.execute_reply.started":"2023-06-08T08:20:46.099134Z","shell.execute_reply":"2023-06-08T08:20:46.111793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open('/kaggle/input/asl-fingerspelling/character_to_prediction_index.json') as json_file:\n    LABEL_DICT = json.load(json_file)\n# LABEL_DICT","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:37:17.19802Z","iopub.execute_input":"2023-06-08T08:37:17.1985Z","iopub.status.idle":"2023-06-08T08:37:17.216931Z","shell.execute_reply.started":"2023-06-08T08:37:17.198457Z","shell.execute_reply":"2023-06-08T08:37:17.215093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_columns = temp.drop(columns=['frame']).columns\n\ndef load_relevant_data_subset(pq_path):\n    return pd.read_parquet(pq_path, columns=selected_columns)\n\ndef encode_strings(string_list, mapping=LABEL_DICT):\n    return [[mapping[char] for char in string] for string in string_list]\n    \nselected_columns","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:37:12.149698Z","iopub.execute_input":"2023-06-08T08:37:12.150178Z","iopub.status.idle":"2023-06-08T08:37:12.503141Z","shell.execute_reply.started":"2023-06-08T08:37:12.150134Z","shell.execute_reply":"2023-06-08T08:37:12.501517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1629/3","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:32:21.662689Z","iopub.execute_input":"2023-06-08T08:32:21.663161Z","iopub.status.idle":"2023-06-08T08:32:21.673203Z","shell.execute_reply.started":"2023-06-08T08:32:21.663117Z","shell.execute_reply":"2023-06-08T08:32:21.671506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load_relevant_data_subset('/kaggle/input/asl-fingerspelling/train_landmarks/1019715464.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:29:21.784527Z","iopub.execute_input":"2023-06-08T08:29:21.785032Z","iopub.status.idle":"2023-06-08T08:29:21.791288Z","shell.execute_reply.started":"2023-06-08T08:29:21.784988Z","shell.execute_reply":"2023-06-08T08:29:21.789603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encode_strings(['asdif'])","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:45:39.055623Z","iopub.execute_input":"2023-06-08T08:45:39.056081Z","iopub.status.idle":"2023-06-08T08:45:39.065133Z","shell.execute_reply.started":"2023-06-08T08:45:39.05604Z","shell.execute_reply":"2023-06-08T08:45:39.063454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_row(row):\n    coordinates = load_relevant_data_subset(f'/kaggle/input/asl-fingerspelling/{row.path}')\n    coordinates = coordinates.loc[coordinates.index==row.sequence_id].values.astype('float32')\n    coordinates_encoded = coordinates.tobytes()\n    participant_id = int(row.participant_id)\n    sequence_id = int(row.sequence_id)\n#     sign = int(LABEL_DICT[row.sign])\n    phrase = row.phrase.encode('utf-8')\n    phrase_encoded = encode_strings([row.phrase])[0]\n    record_bytes = tf.train.Example(features=tf.train.Features(feature={\n                'coordinates': tf.train.Feature(bytes_list=tf.train.BytesList(value=[coordinates_encoded])),\n                'file_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[participant_id])),\n                'participant_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[participant_id])),\n                'sequence_id':tf.train.Feature(int64_list=tf.train.Int64List(value=[sequence_id])),\n                'phrase': tf.train.Feature(bytes_list=tf.train.BytesList(value=[phrase])),\n                'phrase_encoded': tf.train.Feature(int64_list=tf.train.Int64List(value=phrase_encoded)),\n#                 'sign':tf.train.Feature(int64_list=tf.train.Int64List(value=[sign])),\n                })).SerializeToString()\n    return record_bytes\n\ndef process_chunk(chunk, tfrecord_name):\n    options = tf.io.TFRecordOptions(compression_type='GZIP', compression_level=9)\n    with tf.io.TFRecordWriter(tfrecord_name, options=options) as file_writer:\n        for i, row in tqdm(chunk.iterrows()):\n            record_bytes = encode_row(row)\n            file_writer.write(record_bytes)\n            del record_bytes\n        file_writer.close()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:52:07.558142Z","iopub.execute_input":"2023-06-08T08:52:07.559826Z","iopub.status.idle":"2023-06-08T08:52:07.584212Z","shell.execute_reply.started":"2023-06-08T08:52:07.559688Z","shell.execute_reply":"2023-06-08T08:52:07.581599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = train_df.iloc[0]\n# coordinates = load_relevant_data_subset(f'/kaggle/input/asl-fingerspelling/{row.path}')\n# coordinates_encoded = coordinates.tobytes()\n# participant_id = int(row.participant_id)\n# sequence_id = int(row.sequence_id)\n# sign = int(LABEL_DICT[row.sign])\n# encode_row(row)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:52:33.431486Z","iopub.execute_input":"2023-06-08T08:52:33.431953Z","iopub.status.idle":"2023-06-08T08:52:33.439327Z","shell.execute_reply.started":"2023-06-08T08:52:33.43191Z","shell.execute_reply":"2023-06-08T08:52:33.437707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# record_bytes = tf.train.Example(features=tf.train.Features(feature={\n#             'coordinates': tf.train.Feature(bytes_list=tf.train.BytesList(value=[coordinates_encoded])),\n#             'participant_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[participant_id])),\n#             'sequence_id':tf.train.Feature(int64_list=tf.train.Int64List(value=[sequence_id])),\n#             'sign':tf.train.Feature(int64_list=tf.train.Int64List(value=[sign])),\n#             }))","metadata":{"execution":{"iopub.status.busy":"2023-06-08T08:52:41.046026Z","iopub.execute_input":"2023-06-08T08:52:41.046497Z","iopub.status.idle":"2023-06-08T08:52:41.052344Z","shell.execute_reply.started":"2023-06-08T08:52:41.046452Z","shell.execute_reply":"2023-06-08T08:52:41.050882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FILES = len(train_df)\nCHUNK_SIZE = 512\nN_PART = 1\nFOLD = 4\npart = 0\n\nclass CFG:\n    seed = 42\n    n_splits = 5","metadata":{"execution":{"iopub.status.busy":"2023-06-08T09:01:19.403312Z","iopub.execute_input":"2023-06-08T09:01:19.403796Z","iopub.status.idle":"2023-06-08T09:01:19.412087Z","shell.execute_reply.started":"2023-06-08T09:01:19.403747Z","shell.execute_reply":"2023-06-08T09:01:19.410371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_folds = train_df.copy()\ntrain_folds['fold']=-1\n\n# num_bins = 5\n\ntrain_folds = train_folds.sample(frac=1, random_state=CFG.seed).reset_index(drop=True)\n# gkfold = StratifiedGroupKFold(n_splits=CFG.n_splits, shuffle=True, random_state=CFG.seed) \n# print(f'{CFG.n_splits}fold training', len(train_folds), 'samples')\n# for fold_idx, (train_idx, valid_idx) in enumerate(gkfold.split(train_folds, y=train_folds['sign'].values, groups=train_folds.participant_id)):\n#     train_folds.loc[valid_idx,'fold'] = fold_idx\n#     print(f'fold{fold_idx}:', 'train', len(train_idx), 'valid', len(valid_idx))\n# kfold = KFold(n_splits=CFG.n_splits, shuffle=True, random_state=CFG.seed) \n# print(f'{CFG.n_splits}fold training', len(train_folds), 'samples')\n# for fold_idx, (train_idx, valid_idx) in enumerate(kfold.split(train_folds)):\n#     train_folds.loc[valid_idx,'fold'] = fold_idx\n#     print(f'fold{fold_idx}:', 'train', len(train_idx), 'valid', len(valid_idx))\nkfold = GroupKFold(n_splits=CFG.n_splits) \nprint(f'{CFG.n_splits}fold training', len(train_folds), 'samples')\nfor fold_idx, (train_idx, valid_idx) in enumerate(kfold.split(train_folds, groups=train_folds.participant_id)):\n    train_folds.loc[valid_idx,'fold'] = fold_idx\n    print(f'fold{fold_idx}:', 'train', len(train_idx), 'valid', len(valid_idx))\n    \nassert not (train_folds['fold']==-1).sum()\nassert len(np.unique(train_folds['fold']))==CFG.n_splits\ntrain_folds.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T09:01:19.870951Z","iopub.execute_input":"2023-06-08T09:01:19.87141Z","iopub.status.idle":"2023-06-08T09:01:19.936205Z","shell.execute_reply.started":"2023-06-08T09:01:19.871367Z","shell.execute_reply":"2023-06-08T09:01:19.934377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport json\nimport os\n\nDATASET_NAME = f'ASLFR-{CFG.n_splits}fold'\n### Create Kaggle Dataset if not exists \n# DATASET_NAME='nearestls'\n\n!rm -rf /tmp/{DATASET_NAME}\n\nos.makedirs(f'/tmp/{DATASET_NAME}', exist_ok=True)\n\nwith open('/kaggle/input/kaggleapi/kaggle.json') as f:\n    kaggle_creds = json.load(f)\n    \nos.environ['KAGGLE_USERNAME'] = kaggle_creds['username']\nos.environ['KAGGLE_KEY'] = kaggle_creds['key']\n\n!kaggle datasets init -p /tmp/{DATASET_NAME}\n\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json') as f:\n    dataset_meta = json.load(f)\n\ndataset_meta['id'] = f'hoyso48/{DATASET_NAME}'\ndataset_meta['title'] = DATASET_NAME\nwith open(f'/tmp/{DATASET_NAME}/dataset-metadata.json', \"w\") as outfile:\n    json.dump(dataset_meta, outfile)\nprint(dataset_meta)\n\n!cp /tmp/{DATASET_NAME}/dataset-metadata.json /tmp/{DATASET_NAME}/meta.json\n!ls /tmp/{DATASET_NAME}\n\n!kaggle datasets create -p /tmp/{DATASET_NAME} --public","metadata":{"execution":{"iopub.status.busy":"2023-06-08T09:02:26.421361Z","iopub.execute_input":"2023-06-08T09:02:26.422704Z","iopub.status.idle":"2023-06-08T09:02:40.05977Z","shell.execute_reply.started":"2023-06-08T09:02:26.422641Z","shell.execute_reply":"2023-06-08T09:02:40.057864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put every image in a seperate TFRecord file\n# Make Pairs of Views as input to the model\n\n\ndef split_dataframe(df, chunk_size = 10000): \n    chunks = list()\n    num_chunks = len(df) // chunk_size + 1\n    for i in range(num_chunks):\n        chunks.append(df[i*chunk_size:(i+1)*chunk_size])\n    return chunks\n\nfor fold in range(CFG.n_splits):#[FOLD]:#range(CFG.n_splits):\n    rows = train_folds[train_folds['fold']==fold]\n    chunks = split_dataframe(rows, CHUNK_SIZE)\n    part_size = len(chunks)//N_PART\n    last = (part+1)*part_size if part != N_PART - 1 else len(chunks)+1\n    chunks = chunks[part*part_size:last]\n    \n    N = [len(x) for x in chunks]\n    _ = Parallel(n_jobs=cpu_count())(\n        delayed(process_chunk)(x, f'/tmp/{DATASET_NAME}/fold{fold}-{i}-{n}.tfrecords')\n        for i,(x,n) in enumerate(zip(chunks,N))\n    )","metadata":{"execution":{"iopub.status.busy":"2023-06-08T09:02:40.064119Z","iopub.execute_input":"2023-06-08T09:02:40.064738Z","iopub.status.idle":"2023-06-08T09:02:59.717125Z","shell.execute_reply.started":"2023-06-08T09:02:40.064665Z","shell.execute_reply":"2023-06-08T09:02:59.714377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nversion_name = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\nprint(version_name)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T09:02:59.719352Z","iopub.status.idle":"2023-06-08T09:02:59.720363Z","shell.execute_reply.started":"2023-06-08T09:02:59.719887Z","shell.execute_reply":"2023-06-08T09:02:59.719933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!kaggle datasets version -m {version_name} -p /tmp/{DATASET_NAME} -r zip","metadata":{"execution":{"iopub.status.busy":"2023-06-08T09:02:59.724832Z","iopub.status.idle":"2023-06-08T09:02:59.725694Z","shell.execute_reply.started":"2023-06-08T09:02:59.725406Z","shell.execute_reply":"2023-06-08T09:02:59.725435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nprint(KaggleDatasets().get_gcs_path('aslfr-5fold'))","metadata":{"execution":{"iopub.status.busy":"2023-03-22T05:22:51.344956Z","iopub.execute_input":"2023-03-22T05:22:51.346008Z","iopub.status.idle":"2023-03-22T05:23:46.946611Z","shell.execute_reply.started":"2023-03-22T05:22:51.345956Z","shell.execute_reply":"2023-03-22T05:23:46.944712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}