{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net - Record Generation\n\nPrepare an initial set of TF records for a first baseline attempt.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport h5py\nimport tensorflow as tf\nprint(tf.__version__)\nfrom sklearn.model_selection import KFold\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.200983Z","iopub.execute_input":"2022-11-27T13:58:03.201531Z","iopub.status.idle":"2022-11-27T13:58:03.211899Z","shell.execute_reply.started":"2022-11-27T13:58:03.201491Z","shell.execute_reply":"2022-11-27T13:58:03.210539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"code","source":"class Config:\n    # setting\n    seed = 887    \n    n_jobs = 4\n    # input data\n    base_dir = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\"\n    path_train = base_dir + \"train_labels.csv\"\n    train_dir = base_dir + \"train/\"\n    path_test = base_dir + \"sample_submission.csv\"\n    test_dir = base_dir + \"test/\"\n    # output\n    output_dir = \"/kaggle/working/\"\n    output_dir_train = output_dir + \"train/\"\n    output_dir_test = output_dir + \"test/\"\n    name_train = \"g2net-train-orig\"\n    name_test = \"g2net-test\"\n    shard_train = 12\n    shard_test = 80\n    # spectrum\n    ts_max = 4096\n    img_size = 360, 1024\n    #ts_max = 1474560//img_size[0]\n    reshape_size = (*img_size, ts_max // img_size[1])\n    \n\ncfg = Config()\nos.makedirs(cfg.output_dir, exist_ok=True)\nos.makedirs(cfg.output_dir_train, exist_ok=True)\nos.makedirs(cfg.output_dir_test, exist_ok=True)\ncfg","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.214767Z","iopub.execute_input":"2022-11-27T13:58:03.215298Z","iopub.status.idle":"2022-11-27T13:58:03.237516Z","shell.execute_reply.started":"2022-11-27T13:58:03.215255Z","shell.execute_reply":"2022-11-27T13:58:03.236147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper function","metadata":{"execution":{"iopub.status.busy":"2022-11-07T21:10:41.714639Z","iopub.execute_input":"2022-11-07T21:10:41.715065Z","iopub.status.idle":"2022-11-07T21:10:41.721334Z","shell.execute_reply.started":"2022-11-07T21:10:41.715029Z","shell.execute_reply":"2022-11-07T21:10:41.720524Z"}}},{"cell_type":"code","source":"def add_shard(data, n_shards, shuffle=True):\n    \"\"\"Add shard number\"\"\"\n    kf = KFold(n_splits=n_shards, shuffle=shuffle, random_state=cfg.seed if shuffle else None)\n    data[\"shard\"] = -1\n    for i, (_, iloc_test) in enumerate(kf.split(data)):\n        ind_test = data.iloc[iloc_test].index\n        data.loc[ind_test, \"shard\"] = i","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.23968Z","iopub.execute_input":"2022-11-27T13:58:03.240105Z","iopub.status.idle":"2022-11-27T13:58:03.247594Z","shell.execute_reply.started":"2022-11-27T13:58:03.240067Z","shell.execute_reply":"2022-11-27T13:58:03.246348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_arr(arr):\n    num_freq, num_ts = arr.shape\n    t = np.zeros(shape=(num_freq, cfg.ts_max), dtype=arr.dtype)\n    num_ts = min(num_ts, cfg.ts_max)\n    t[:, :num_ts] = arr[:, :num_ts]\n    # power\n    t = t.real ** 2 + t.imag ** 2\n    # normalize by mean over all entries\n    t = t / np.mean(t)\n    # group neighboring timestamps in 3rd dimension\n    t = t.reshape((cfg.reshape_size))\n    # take average over 3rd dimension (neighboring ts)\n    img = np.mean(t, axis=2)\n    return img\n\n\ndef load_spectra(path_file):\n    with h5py.File(path_file, 'r') as f:\n        rec_id = list(f.keys())[0]\n        h1 = np.array(f[f\"{rec_id}/H1/SFTs\"]) * 1e22\n        h1 = process_arr(h1)\n        l1 = np.array(f[f\"{rec_id}/L1/SFTs\"]) * 1e22\n        l1 = process_arr(l1)\n    return np.asarray([h1, l1]).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.249035Z","iopub.execute_input":"2022-11-27T13:58:03.249899Z","iopub.status.idle":"2022-11-27T13:58:03.260536Z","shell.execute_reply.started":"2022-11-27T13:58:03.249861Z","shell.execute_reply":"2022-11-27T13:58:03.25954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\n# def _float_feature(value):\n#     \"\"\"Returns a float_list from a float / double.\"\"\"\n#     return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef to_records(img_bytes, id_bytes, target_int):\n    feature = {\n        'img': _bytes_feature(img_bytes),\n        'id': _bytes_feature(id_bytes),\n    }\n    if target_int is not None:\n        feature['target'] = _int64_feature(target_int)\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef create_record(row, input_dir):\n    path_file = input_dir + f\"{row.id}.hdf5\"\n    img = load_spectra(path_file)\n    rec = to_records(\n        img_bytes=tf.io.serialize_tensor(img).numpy(),\n        id_bytes=row.id.encode('utf-8'),\n        target_int=getattr(row, \"target\", None),\n    )\n    return rec","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.263642Z","iopub.execute_input":"2022-11-27T13:58:03.264109Z","iopub.status.idle":"2022-11-27T13:58:03.283639Z","shell.execute_reply.started":"2022-11-27T13:58:03.264045Z","shell.execute_reply":"2022-11-27T13:58:03.282219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_shard(df, shard, input_dir, output_dir, name):\n    fname = f\"{name}_shard-{shard}_size-{len(df)}.tfrec\"\n    path_output = os.path.join(output_dir, fname)\n    print(f\"shard {shard}: {path_output}\")\n    cnt = 0\n    with tf.io.TFRecordWriter(path_output) as f:\n        for row in df.itertuples():\n            rec = create_record(row, input_dir)\n            f.write(rec.SerializeToString())\n            cnt += 1\n    return cnt","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.285406Z","iopub.execute_input":"2022-11-27T13:58:03.285915Z","iopub.status.idle":"2022-11-27T13:58:03.301572Z","shell.execute_reply.started":"2022-11-27T13:58:03.285863Z","shell.execute_reply":"2022-11-27T13:58:03.300071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create record","metadata":{}},{"cell_type":"markdown","source":"## Training dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(cfg.path_train)\ntrain = train[train[\"target\"] >= 0]\nadd_shard(train, cfg.shard_train)\ntrain = train.sample(frac=1)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.303363Z","iopub.execute_input":"2022-11-27T13:58:03.3039Z","iopub.status.idle":"2022-11-27T13:58:03.380499Z","shell.execute_reply.started":"2022-11-27T13:58:03.303848Z","shell.execute_reply":"2022-11-27T13:58:03.379306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sort_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:03.382269Z","iopub.execute_input":"2022-11-27T13:58:03.382777Z","iopub.status.idle":"2022-11-27T13:58:03.399284Z","shell.execute_reply.started":"2022-11-27T13:58:03.382729Z","shell.execute_reply":"2022-11-27T13:58:03.397787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_args = (cfg.train_dir, cfg.output_dir_train, cfg.name_train)\nresults = Parallel(n_jobs=cfg.n_jobs, verbose=1, backend='threading')(\n    delayed(write_shard)(df, shard, *train_args) for shard, df in train.groupby(\"shard\")\n)\ntrain.sort_index().to_csv(cfg.output_dir + \"/summary_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:16.940116Z","iopub.execute_input":"2022-11-27T13:58:16.940588Z","iopub.status.idle":"2022-11-27T13:58:18.169222Z","shell.execute_reply.started":"2022-11-27T13:58:16.940538Z","shell.execute_reply":"2022-11-27T13:58:18.168008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test dataset","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(cfg.path_test)\ndel test[\"target\"]\nadd_shard(test, cfg.shard_test, shuffle=False)\ntest","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:05.055452Z","iopub.execute_input":"2022-11-27T13:58:05.056018Z","iopub.status.idle":"2022-11-27T13:58:05.141872Z","shell.execute_reply.started":"2022-11-27T13:58:05.055975Z","shell.execute_reply":"2022-11-27T13:58:05.140579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_args = (cfg.test_dir, cfg.output_dir_test, cfg.name_test)\nresults = Parallel(n_jobs=cfg.n_jobs, verbose=1, backend='threading')(\n    delayed(write_shard)(df, shard, *test_args) for shard, df in test.groupby(\"shard\")\n)\ntest.sort_index().to_csv(cfg.output_dir + \"/summary_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:05.145611Z","iopub.execute_input":"2022-11-27T13:58:05.145999Z","iopub.status.idle":"2022-11-27T13:58:06.096507Z","shell.execute_reply.started":"2022-11-27T13:58:05.145968Z","shell.execute_reply":"2022-11-27T13:58:06.093516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"code","source":"!tree $cfg.output_dir","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:06.098704Z","iopub.execute_input":"2022-11-27T13:58:06.09911Z","iopub.status.idle":"2022-11-27T13:58:07.27324Z","shell.execute_reply.started":"2022-11-27T13:58:06.099046Z","shell.execute_reply":"2022-11-27T13:58:07.271858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/2104080/how-do-i-check-file-size-in-python\n\ndef convert_bytes(num):\n    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:\n        if num < 1024.0:\n            return \"%3.1f %s\" % (num, x)\n        num /= 1024.0","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:07.27531Z","iopub.execute_input":"2022-11-27T13:58:07.275895Z","iopub.status.idle":"2022-11-27T13:58:07.284502Z","shell.execute_reply.started":"2022-11-27T13:58:07.275842Z","shell.execute_reply":"2022-11-27T13:58:07.283389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for d in [cfg.output_dir_train, cfg.output_dir_test]:\n    bs = sum(os.stat(f).st_size for f in pathlib.Path(d).glob(\"*.*\"))\n    print(d, convert_bytes(bs))","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:58:07.288729Z","iopub.execute_input":"2022-11-27T13:58:07.289162Z","iopub.status.idle":"2022-11-27T13:58:07.299365Z","shell.execute_reply.started":"2022-11-27T13:58:07.289125Z","shell.execute_reply":"2022-11-27T13:58:07.297568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}