{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Weird property of G2Net competition data\n \nParts of code copied from https://www.kaggle.com/code/ayuraj/g2net-understand-the-data","metadata":{}},{"cell_type":"markdown","source":"This notebook was intended to illustrate my doubts about one aspect of competition data. Now, everything was clarified by organizers in [this discussion](https://www.kaggle.com/competitions/g2net-detecting-continuous-gravitational-waves/discussion/363460), so you should not care about this notebook anymore :-)","metadata":{}},{"cell_type":"code","source":"# some basic imports, nothing unusual\n\nimport os\nimport h5py # import to read hdf5\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom pathlib import Path\nimport re\nimport matplotlib.pyplot as plt\nimport tqdm\n%matplotlib inline\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-01T17:06:10.986564Z","iopub.execute_input":"2022-11-01T17:06:10.986937Z","iopub.status.idle":"2022-11-01T17:06:11.006258Z","shell.execute_reply.started":"2022-11-01T17:06:10.986908Z","shell.execute_reply":"2022-11-01T17:06:11.005212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# simple function to read single data file, taken from https://www.kaggle.com/code/ayuraj/g2net-understand-the-data\ndef read_data(file: Path):\n    with h5py.File(file, \"r\") as f:\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        freq_hz = list(f[\"frequency_Hz\"])\n        \n        h1_stft = h1[\"SFTs\"][()]\n        h1_timestamp = h1[\"timestamps_GPS\"][()]\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        l1_timestamp = l1[\"timestamps_GPS\"][()]\n        \n        return {\n            \"H1\": [h1_stft, h1_timestamp],\n            \"L1\": [l1_stft, l1_timestamp],\n            \"freq_hz\": freq_hz\n        }\n","metadata":{"execution":{"iopub.status.busy":"2022-11-01T16:54:06.715231Z","iopub.execute_input":"2022-11-01T16:54:06.715578Z","iopub.status.idle":"2022-11-01T16:54:06.723621Z","shell.execute_reply.started":"2022-11-01T16:54:06.715551Z","shell.execute_reply":"2022-11-01T16:54:06.722771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/g2net-detecting-continuous-gravitational-waves'\nassert os.path.isdir(ROOT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T16:56:07.078275Z","iopub.execute_input":"2022-11-01T16:56:07.078593Z","iopub.status.idle":"2022-11-01T16:56:07.084408Z","shell.execute_reply.started":"2022-11-01T16:56:07.078568Z","shell.execute_reply":"2022-11-01T16:56:07.083433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mapping of file ids to target values \ndf = pd.read_csv(os.path.join(ROOT_DIR, 'train_labels.csv'))\nid_to_target = {row.id: row.target for _, row in df.iterrows()}\nprint(len(id_to_target))","metadata":{"execution":{"iopub.status.busy":"2022-11-01T16:56:09.695913Z","iopub.execute_input":"2022-11-01T16:56:09.69629Z","iopub.status.idle":"2022-11-01T16:56:09.75598Z","shell.execute_reply.started":"2022-11-01T16:56:09.69626Z","shell.execute_reply":"2022-11-01T16:56:09.754954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first weird thing: if we compute std of spectrograms in complex64, most of them\n# have the same value of 1.4973568266036865e-22\ndpath = os.path.join(ROOT_DIR, 'train')\nfnames = os.listdir(dpath)\nregex = re.compile('^[0-9a-f]{9}\\.hdf5$')\ncount_weird = 0\ncount_other = 0\nweird_value = 1.4973568266036865e-22\nfor fname in tqdm.tqdm(fnames):\n    if regex.match(fname):\n        fpath = os.path.join(dpath, fname)\n        data = read_data(Path(fpath))\n        sft_h = data['H1'][0]\n        sft_l = data['L1'][0]\n        if np.std(sft_h) == np.std(sft_l) == weird_value:\n            count_weird += 1\n        else:\n            count_other +=1\nprint(f'{count_weird} inputs out of total {count_weird+count_other} has the same std')","metadata":{"execution":{"iopub.status.busy":"2022-11-01T17:15:48.570793Z","iopub.execute_input":"2022-11-01T17:15:48.571161Z","iopub.status.idle":"2022-11-01T17:18:06.191341Z","shell.execute_reply.started":"2022-11-01T17:15:48.571132Z","shell.execute_reply":"2022-11-01T17:18:06.190465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# second weird thing: if we compute std of spectrograms in complex128,\n# they are not equal anymore ... and they starts to be \n# 'not so wrong' indicator of target label\npositive = list()\nnegative = list()\nfor fname in tqdm.tqdm(fnames):\n    if regex.match(fname):\n        fileid = fname[:-5]\n        assert fileid in id_to_target\n        fpath = os.path.join(dpath, fname)\n        data = read_data(Path(fpath))\n        sft_h = data['H1'][0]\n        sft_l = data['L1'][0]\n        std_h = np.std(sft_h.astype(np.complex128))\n        std_l = np.std(sft_l.astype(np.complex128))\n        if np.abs(std_h - 1.5e-22) < 1.0e-22:\n            if id_to_target[fileid]:\n                positive.append((std_h, std_l))\n            else:\n                negative.append((std_h, std_l))\nplt.scatter([h for h, l in positive], [l for h, l in positive], c='g')\nplt.scatter([h for h, l in negative], [l for h, l in negative], c='r')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-01T17:18:28.353199Z","iopub.execute_input":"2022-11-01T17:18:28.354353Z","iopub.status.idle":"2022-11-01T17:20:55.310154Z","shell.execute_reply.started":"2022-11-01T17:18:28.354318Z","shell.execute_reply":"2022-11-01T17:20:55.308623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I started [discussion](https://www.kaggle.com/competitions/g2net-detecting-continuous-gravitational-waves/discussion/363460) about this, because for me it looks almost like target label leak. But maybe I'm wrong :-)","metadata":{}}]}