{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net - Record Generation\n\nPrepare an initial set of TF records for a first baseline attempt.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport h5py\nimport tensorflow as tf\nprint(tf.__version__)\nfrom sklearn.model_selection import KFold\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:38:34.414253Z","iopub.execute_input":"2022-11-10T19:38:34.414816Z","iopub.status.idle":"2022-11-10T19:38:41.152634Z","shell.execute_reply.started":"2022-11-10T19:38:34.414709Z","shell.execute_reply":"2022-11-10T19:38:41.151298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"code","source":"class Config:\n    # setting\n    seed = 887    \n    n_jobs = 4\n    # input data\n    base_dir = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\"\n    path_train = base_dir + \"train_labels.csv\"\n    train_dir = base_dir + \"train/\"\n    path_test = base_dir + \"sample_submission.csv\"\n    test_dir = base_dir + \"test/\"\n    # output\n    output_dir = \"/kaggle/working/\"\n    output_dir_train = output_dir + \"train/\"\n    output_dir_test = output_dir + \"test/\"\n    name_train = \"g2net-train-orig\"\n    name_test = \"g2net-test\"\n    shard_train = 12\n    shard_test = 80\n    # spectrum\n    ts_max = 4096\n    img_size = 360, 128\n    reshape_size = (*img_size, ts_max // img_size[1])\n    \n\ncfg = Config()\nos.makedirs(cfg.output_dir, exist_ok=True)\nos.makedirs(cfg.output_dir_train, exist_ok=True)\nos.makedirs(cfg.output_dir_test, exist_ok=True)\ncfg","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:38:41.156861Z","iopub.execute_input":"2022-11-10T19:38:41.157757Z","iopub.status.idle":"2022-11-10T19:38:41.172417Z","shell.execute_reply.started":"2022-11-10T19:38:41.157713Z","shell.execute_reply":"2022-11-10T19:38:41.171194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper function","metadata":{"execution":{"iopub.status.busy":"2022-11-07T21:10:41.714639Z","iopub.execute_input":"2022-11-07T21:10:41.715065Z","iopub.status.idle":"2022-11-07T21:10:41.721334Z","shell.execute_reply.started":"2022-11-07T21:10:41.715029Z","shell.execute_reply":"2022-11-07T21:10:41.720524Z"}}},{"cell_type":"code","source":"def add_shard(data, n_shards, shuffle=True):\n    \"\"\"Add shard number\"\"\"\n    kf = KFold(n_splits=n_shards, shuffle=shuffle, random_state=cfg.seed if shuffle else None)\n    data[\"shard\"] = -1\n    for i, (_, iloc_test) in enumerate(kf.split(data)):\n        ind_test = data.iloc[iloc_test].index\n        data.loc[ind_test, \"shard\"] = i","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:38:41.17395Z","iopub.execute_input":"2022-11-10T19:38:41.174361Z","iopub.status.idle":"2022-11-10T19:38:41.182287Z","shell.execute_reply.started":"2022-11-10T19:38:41.174329Z","shell.execute_reply":"2022-11-10T19:38:41.181046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_arr(arr):\n    num_freq, num_ts = arr.shape\n    t = np.zeros(shape=(num_freq, cfg.ts_max), dtype=arr.dtype)\n    num_ts = min(num_ts, cfg.ts_max)\n    t[:, :num_ts] = arr[:, :num_ts]\n    # power\n    t = t.real ** 2 + t.imag ** 2\n    # normalize by mean over all entries\n    t = t / np.mean(t)\n    # group neighboring timestamps in 3rd dimension\n    t = t.reshape((cfg.reshape_size))\n    # take average over 3rd dimension (neighboring ts)\n    img = np.mean(t, axis=2)\n    return img\n\n\ndef load_spectra(path_file):\n    with h5py.File(path_file, 'r') as f:\n        rec_id = list(f.keys())[0]\n        h1 = np.array(f[f\"{rec_id}/H1/SFTs\"]) * 1e22\n        h1 = process_arr(h1)\n        l1 = np.array(f[f\"{rec_id}/L1/SFTs\"]) * 1e22\n        l1 = process_arr(l1)\n    return np.asarray([h1, l1]).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:38:41.184631Z","iopub.execute_input":"2022-11-10T19:38:41.184996Z","iopub.status.idle":"2022-11-10T19:38:41.197157Z","shell.execute_reply.started":"2022-11-10T19:38:41.184964Z","shell.execute_reply":"2022-11-10T19:38:41.195821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\n# def _float_feature(value):\n#     \"\"\"Returns a float_list from a float / double.\"\"\"\n#     return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef to_records(img_bytes, id_bytes, target_int):\n    feature = {\n        'img': _bytes_feature(img_bytes),\n        'id': _bytes_feature(id_bytes),\n    }\n    if target_int is not None:\n        feature['target'] = _int64_feature(target_int)\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef create_record(row, input_dir):\n    path_file = input_dir + f\"{row.id}.hdf5\"\n    img = load_spectra(path_file)\n    rec = to_records(\n        img_bytes=tf.io.serialize_tensor(img).numpy(),\n        id_bytes=row.id.encode('utf-8'),\n        target_int=getattr(row, \"target\", None),\n    )\n    return rec","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:38:41.198513Z","iopub.execute_input":"2022-11-10T19:38:41.198897Z","iopub.status.idle":"2022-11-10T19:38:41.218647Z","shell.execute_reply.started":"2022-11-10T19:38:41.198865Z","shell.execute_reply":"2022-11-10T19:38:41.216961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_shard(df, shard, input_dir, output_dir, name):\n    fname = f\"{name}_shard-{shard}_size-{len(df)}.tfrec\"\n    path_output = os.path.join(output_dir, fname)\n    print(f\"shard {shard}: {path_output}\")\n    cnt = 0\n    with tf.io.TFRecordWriter(path_output) as f:\n        for row in df.itertuples():\n            rec = create_record(row, input_dir)\n            f.write(rec.SerializeToString())\n            cnt += 1\n    return cnt","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:38:41.220418Z","iopub.execute_input":"2022-11-10T19:38:41.221014Z","iopub.status.idle":"2022-11-10T19:38:41.230972Z","shell.execute_reply.started":"2022-11-10T19:38:41.220975Z","shell.execute_reply":"2022-11-10T19:38:41.229831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create record","metadata":{}},{"cell_type":"markdown","source":"## Training dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(cfg.path_train)\ntrain = train[train[\"target\"] >= 0]\nadd_shard(train, cfg.shard_train)\ntrain = train.sample(frac=1)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:39:44.974302Z","iopub.execute_input":"2022-11-10T19:39:44.974704Z","iopub.status.idle":"2022-11-10T19:39:45.00505Z","shell.execute_reply.started":"2022-11-10T19:39:44.974673Z","shell.execute_reply":"2022-11-10T19:39:45.00414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sort_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:40:02.63465Z","iopub.execute_input":"2022-11-10T19:40:02.635039Z","iopub.status.idle":"2022-11-10T19:40:02.656545Z","shell.execute_reply.started":"2022-11-10T19:40:02.635009Z","shell.execute_reply":"2022-11-10T19:40:02.655343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_args = (cfg.train_dir, cfg.output_dir_train, cfg.name_train)\nresults = Parallel(n_jobs=cfg.n_jobs, verbose=1, backend='threading')(\n    delayed(write_shard)(df, shard, *train_args) for shard, df in train.groupby(\"shard\")\n)\ntrain.sort_index().to_csv(cfg.output_dir + \"/summary_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:11:09.135207Z","iopub.execute_input":"2022-11-09T22:11:09.136328Z","iopub.status.idle":"2022-11-09T22:11:34.958973Z","shell.execute_reply.started":"2022-11-09T22:11:09.136268Z","shell.execute_reply":"2022-11-09T22:11:34.957406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test dataset","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(cfg.path_test)\ndel test[\"target\"]\nadd_shard(test, cfg.shard_test, shuffle=False)\ntest","metadata":{"execution":{"iopub.status.busy":"2022-11-10T19:40:20.997925Z","iopub.execute_input":"2022-11-10T19:40:20.998447Z","iopub.status.idle":"2022-11-10T19:40:21.082176Z","shell.execute_reply.started":"2022-11-10T19:40:20.99841Z","shell.execute_reply":"2022-11-10T19:40:21.080864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_args = (cfg.test_dir, cfg.output_dir_test, cfg.name_test)\nresults = Parallel(n_jobs=cfg.n_jobs, verbose=1, backend='threading')(\n    delayed(write_shard)(df, shard, *test_args) for shard, df in test.groupby(\"shard\")\n)\ntest.sort_index().to_csv(cfg.output_dir + \"/summary_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:12:16.134953Z","iopub.execute_input":"2022-11-09T22:12:16.135371Z","iopub.status.idle":"2022-11-09T22:13:33.094112Z","shell.execute_reply.started":"2022-11-09T22:12:16.13534Z","shell.execute_reply":"2022-11-09T22:13:33.092061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"code","source":"!tree $cfg.output_dir","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:13:50.526293Z","iopub.execute_input":"2022-11-09T22:13:50.528067Z","iopub.status.idle":"2022-11-09T22:13:51.698374Z","shell.execute_reply.started":"2022-11-09T22:13:50.527985Z","shell.execute_reply":"2022-11-09T22:13:51.697136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/2104080/how-do-i-check-file-size-in-python\n\ndef convert_bytes(num):\n    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:\n        if num < 1024.0:\n            return \"%3.1f %s\" % (num, x)\n        num /= 1024.0","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:13:58.9157Z","iopub.execute_input":"2022-11-09T22:13:58.916249Z","iopub.status.idle":"2022-11-09T22:13:58.924127Z","shell.execute_reply.started":"2022-11-09T22:13:58.916204Z","shell.execute_reply":"2022-11-09T22:13:58.922884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for d in [cfg.output_dir_train, cfg.output_dir_test]:\n    bs = sum(os.stat(f).st_size for f in pathlib.Path(d).glob(\"*.*\"))\n    print(d, convert_bytes(bs))","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:14:27.102558Z","iopub.execute_input":"2022-11-09T22:14:27.103021Z","iopub.status.idle":"2022-11-09T22:14:27.112531Z","shell.execute_reply.started":"2022-11-09T22:14:27.102988Z","shell.execute_reply":"2022-11-09T22:14:27.110988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}