{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This Notebook\n\nThis notebook is a modified version of [George Chirita's notebook](https://www.kaggle.com/code/crischir/consolidatedg2netdataset/notebook) used for the G2Net continuous gravitational waves task. It consolidates Chirita's multiple synthetic datasets and the original G2Net data into a single dataset, and improves the signal-to-noise ratio of the resulting dataset using the stem from [laeyoung's notebook](https://www.kaggle.com/code/laeyoung/g2net-large-kernel-inference). The final data is intended to be used by [our other notebook](https://www.kaggle.com/code/hnsyprst/publish-denoised-data-soap-cnn), **which includes a visualisation of the noise reduced spectrograms extracted by this notebook**. For speed, the code for generating these visualisations has note been replicated here; to see this code, please view our other notebook.\n\nThe dataset provided by G2Net for the competition exhibits severe class imbalance, as found by various EDA notebooks. The training set is also very small. To increase the quantity of data for training,[ one of the competition hosts has suggested generating additional samples](https://www.kaggle.com/competitions/g2net-detecting-continuous-gravitational-waves/discussion/347052). Chirita has generously made their synthetic datasets public, and has provided [a notebook](https://www.kaggle.com/code/crischir/consolidatedg2netdataset/notebook) for consolidating their data and the original G2Net data into a single dataset, as mentioned above.\n\nLaeyoung demonstrated that using very large convolutional kernels helps CNNs to extract large-scale signal shapes from the spectrograms provided by G2Net in [their notebook](https://www.kaggle.com/code/laeyoung/g2net-large-kernel-inference). \n\nThis notebook extends Chirita's notebook, removing the compression they apply to the consolidated dataset and using the stem from laeyoung's model to extract spectrograms with reduced noise from the consolidated dataset.","metadata":{}},{"cell_type":"code","source":"import h5py\nfrom glob import glob\nfrom pathlib import Path\nimport os\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\n\nfrom pathlib import Path\nimport shutil\n\n# Providing the folder path\norigin = '/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-04T11:43:06.95985Z","iopub.execute_input":"2023-01-04T11:43:06.960596Z","iopub.status.idle":"2023-01-04T11:43:07.171809Z","shell.execute_reply.started":"2023-01-04T11:43:06.960473Z","shell.execute_reply":"2023-01-04T11:43:07.170381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Output\n(with Original Data for Comparison)","metadata":{}},{"cell_type":"code","source":"def display_data(H1, L1, spect):\n    fig = plt.figure(figsize=(16,12), constrained_layout=True)\n    \n    subfigs = fig.subfigures(nrows=3, ncols=1)\n    subfigs[0].suptitle('Spectrograms before Feature Extraction', fontsize=16)\n    axs_0 = subfigs[0].subplots(nrows=1, ncols=1)\n    axs_0.imshow(H1.T,aspect=\"auto\",origin=\"lower\",cmap=\"gray\")\n    axs_0.set_title('H1 Spectrogram')\n    \n    axs_1 = subfigs[1].subplots(nrows=1, ncols=1)\n    axs_1.imshow(L1.T,aspect=\"auto\",origin=\"lower\",cmap=\"gray\")\n    axs_1.set_title('L1 Spectrogram')\n    \n    subfigs[2].suptitle('Feature Extracted by this Notebook', fontsize=16)\n    axs_2 = subfigs[2].subplots(nrows=1, ncols=1)\n    axs_2.imshow(spect.T,aspect=\"auto\",origin=\"lower\",cmap=\"gray\")\n    \nexample_H1 = np.load('/kaggle/input/g2net-examples/example_original_H1.npy')\nexample_L1 = np.load('/kaggle/input/g2net-examples/example_original_L1.npy')\nexample_spect = np.load('/kaggle/input/g2net-examples/example_spect.npy')\n\ndisplay_data(example_H1, example_L1, example_spect)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T12:05:28.02719Z","iopub.execute_input":"2023-01-04T12:05:28.027654Z","iopub.status.idle":"2023-01-04T12:05:29.996738Z","shell.execute_reply.started":"2023-01-04T12:05:28.027608Z","shell.execute_reply":"2023-01-04T12:05:29.995539Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor root, folders, filenames in os.walk('/kaggle/input'):\n   print(root, folders)\n\ntrain=pd.read_csv(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/train_labels.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:08.666378Z","iopub.execute_input":"2023-01-02T14:12:08.66672Z","iopub.status.idle":"2023-01-02T14:12:16.202849Z","shell.execute_reply.started":"2023-01-02T14:12:08.666686Z","shell.execute_reply":"2023-01-02T14:12:16.201253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['cale_fisier']='/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'+train['id']+'.hdf5'\ntrain.cale_fisier[555]","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.284008Z","iopub.execute_input":"2023-01-02T14:12:16.28455Z","iopub.status.idle":"2023-01-02T14:12:16.29711Z","shell.execute_reply.started":"2023-01-02T14:12:16.284512Z","shell.execute_reply":"2023-01-02T14:12:16.296011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add datasets","metadata":{}},{"cell_type":"markdown","source":"### g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS1'\nadd_on = '/kaggle/input/g2netgaussian-noise-dataset/noise/'\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.335285Z","iopub.execute_input":"2023-01-02T14:12:16.338559Z","iopub.status.idle":"2023-01-02T14:12:16.354515Z","shell.execute_reply.started":"2023-01-02T14:12:16.338523Z","shell.execute_reply":"2023-01-02T14:12:16.353574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS1'\nadd_on = '/kaggle/input/g2netgaussian-noise-dataset/signal/'\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.358737Z","iopub.execute_input":"2023-01-02T14:12:16.359663Z","iopub.status.idle":"2023-01-02T14:12:16.374202Z","shell.execute_reply.started":"2023-01-02T14:12:16.359625Z","shell.execute_reply":"2023-01-02T14:12:16.37311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.378471Z","iopub.execute_input":"2023-01-02T14:12:16.379262Z","iopub.status.idle":"2023-01-02T14:12:16.394821Z","shell.execute_reply.started":"2023-01-02T14:12:16.379221Z","shell.execute_reply":"2023-01-02T14:12:16.39348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g2netnonstationary-dataset","metadata":{}},{"cell_type":"code","source":"print(os.listdir(\"../input\"))","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.3963Z","iopub.execute_input":"2023-01-02T14:12:16.396656Z","iopub.status.idle":"2023-01-02T14:12:16.404379Z","shell.execute_reply.started":"2023-01-02T14:12:16.396622Z","shell.execute_reply":"2023-01-02T14:12:16.403059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS2'\nadd_on = '/kaggle/input/g2netnonstationary-dataset/signal/'\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.406432Z","iopub.execute_input":"2023-01-02T14:12:16.406907Z","iopub.status.idle":"2023-01-02T14:12:16.420989Z","shell.execute_reply.started":"2023-01-02T14:12:16.406871Z","shell.execute_reply":"2023-01-02T14:12:16.420003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS2'\nadd_on = \"/kaggle/input/g2netnonstationary-dataset/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.424583Z","iopub.execute_input":"2023-01-02T14:12:16.424871Z","iopub.status.idle":"2023-01-02T14:12:16.440814Z","shell.execute_reply.started":"2023-01-02T14:12:16.424844Z","shell.execute_reply":"2023-01-02T14:12:16.439875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g2net-datasetnarrowartifacts","metadata":{}},{"cell_type":"code","source":"dataset_number='GS3'\nadd_on = '/kaggle/input/g2net-datasetnarrowartifacts/noise/'\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.442422Z","iopub.execute_input":"2023-01-02T14:12:16.443162Z","iopub.status.idle":"2023-01-02T14:12:16.45686Z","shell.execute_reply.started":"2023-01-02T14:12:16.443115Z","shell.execute_reply":"2023-01-02T14:12:16.455723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### addon-g2netnonstationary-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS4'\nadd_on = \"/kaggle/input/addon-g2netnonstationary-dataset/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.458889Z","iopub.execute_input":"2023-01-02T14:12:16.459644Z","iopub.status.idle":"2023-01-02T14:12:16.473454Z","shell.execute_reply.started":"2023-01-02T14:12:16.459604Z","shell.execute_reply":"2023-01-02T14:12:16.472441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS4'\nadd_on = \"/kaggle/input/addon-g2netnonstationary-dataset/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.474988Z","iopub.execute_input":"2023-01-02T14:12:16.475721Z","iopub.status.idle":"2023-01-02T14:12:16.489877Z","shell.execute_reply.started":"2023-01-02T14:12:16.475678Z","shell.execute_reply":"2023-01-02T14:12:16.488808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### addon-g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS5'\nadd_on = \"/kaggle/input/addon-g2netgaussian-noise-dataset/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.491491Z","iopub.execute_input":"2023-01-02T14:12:16.492254Z","iopub.status.idle":"2023-01-02T14:12:16.506593Z","shell.execute_reply.started":"2023-01-02T14:12:16.492213Z","shell.execute_reply":"2023-01-02T14:12:16.505477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS5'\nadd_on = \"/kaggle/input/addon-g2netgaussian-noise-dataset/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.508235Z","iopub.execute_input":"2023-01-02T14:12:16.508953Z","iopub.status.idle":"2023-01-02T14:12:16.522626Z","shell.execute_reply.started":"2023-01-02T14:12:16.50891Z","shell.execute_reply":"2023-01-02T14:12:16.521736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### add-on-narrow-instrumental-artifacts","metadata":{}},{"cell_type":"code","source":"dataset_number='GS5'\nadd_on = \"/kaggle/input/add-on-narrow-instrumental-artifacts-g2net-dataset/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.524163Z","iopub.execute_input":"2023-01-02T14:12:16.524839Z","iopub.status.idle":"2023-01-02T14:12:16.538723Z","shell.execute_reply.started":"2023-01-02T14:12:16.524799Z","shell.execute_reply":"2023-01-02T14:12:16.537619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2nd-addon-g2netgaussian-noise-dataset\n\n","metadata":{}},{"cell_type":"code","source":"dataset_number='GS6'\nadd_on = \"/kaggle/input/2nd-addon-g2netgaussian-noise-dataset/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.540887Z","iopub.execute_input":"2023-01-02T14:12:16.541636Z","iopub.status.idle":"2023-01-02T14:12:16.556488Z","shell.execute_reply.started":"2023-01-02T14:12:16.541596Z","shell.execute_reply":"2023-01-02T14:12:16.555598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS6'\nadd_on = \"/kaggle/input/2nd-addon-g2netgaussian-noise-dataset/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.558004Z","iopub.execute_input":"2023-01-02T14:12:16.558707Z","iopub.status.idle":"2023-01-02T14:12:16.572326Z","shell.execute_reply.started":"2023-01-02T14:12:16.558668Z","shell.execute_reply":"2023-01-02T14:12:16.571422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3rdaddon-g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS7'\nadd_on = \"/kaggle/input/3rdaddon-g2netgaussian-noise-dataset/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.573839Z","iopub.execute_input":"2023-01-02T14:12:16.574531Z","iopub.status.idle":"2023-01-02T14:12:16.588466Z","shell.execute_reply.started":"2023-01-02T14:12:16.574491Z","shell.execute_reply":"2023-01-02T14:12:16.587394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS7'\nadd_on = \"/kaggle/input/3rdaddon-g2netgaussian-noise-dataset/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.590117Z","iopub.execute_input":"2023-01-02T14:12:16.59084Z","iopub.status.idle":"2023-01-02T14:12:16.605995Z","shell.execute_reply.started":"2023-01-02T14:12:16.590799Z","shell.execute_reply":"2023-01-02T14:12:16.605112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2ndgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS9'\nadd_on = \"/kaggle/input/2ndgeneratinglowsnrgravitywaves/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.642412Z","iopub.execute_input":"2023-01-02T14:12:16.642922Z","iopub.status.idle":"2023-01-02T14:12:16.660954Z","shell.execute_reply.started":"2023-01-02T14:12:16.642886Z","shell.execute_reply":"2023-01-02T14:12:16.659838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS9'\nadd_on = \"/kaggle/input/2ndgeneratinglowsnrgravitywaves/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.665092Z","iopub.execute_input":"2023-01-02T14:12:16.665498Z","iopub.status.idle":"2023-01-02T14:12:16.682508Z","shell.execute_reply.started":"2023-01-02T14:12:16.665464Z","shell.execute_reply":"2023-01-02T14:12:16.681381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### generatinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS10'\nadd_on = \"/kaggle/input/generatinglowsnrgravitywaves/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.686661Z","iopub.execute_input":"2023-01-02T14:12:16.687146Z","iopub.status.idle":"2023-01-02T14:12:16.703904Z","shell.execute_reply.started":"2023-01-02T14:12:16.687106Z","shell.execute_reply":"2023-01-02T14:12:16.702783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS10'\nadd_on = \"/kaggle/input/generatinglowsnrgravitywaves/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.708215Z","iopub.execute_input":"2023-01-02T14:12:16.70871Z","iopub.status.idle":"2023-01-02T14:12:16.725114Z","shell.execute_reply.started":"2023-01-02T14:12:16.708675Z","shell.execute_reply":"2023-01-02T14:12:16.723976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### generatinglowsnrnonstationarygravitywaves\n\n","metadata":{}},{"cell_type":"code","source":"dataset_number='GS10'\nadd_on = \"/kaggle/input/generatinglowsnrnonstationarygravitywaves/noise/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=0\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.729749Z","iopub.execute_input":"2023-01-02T14:12:16.730135Z","iopub.status.idle":"2023-01-02T14:12:16.747125Z","shell.execute_reply.started":"2023-01-02T14:12:16.730099Z","shell.execute_reply":"2023-01-02T14:12:16.746193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS10'\nadd_on = \"/kaggle/input/generatinglowsnrnonstationarygravitywaves/signal/\"\nfiles_add_on = os.listdir(add_on)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\ndf_add['target']=1\ndf_add['cale_fisier']=add_on+df_add.id\ntrain=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.748481Z","iopub.execute_input":"2023-01-02T14:12:16.749185Z","iopub.status.idle":"2023-01-02T14:12:16.763701Z","shell.execute_reply.started":"2023-01-02T14:12:16.749143Z","shell.execute_reply":"2023-01-02T14:12:16.762591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.764979Z","iopub.execute_input":"2023-01-02T14:12:16.765591Z","iopub.status.idle":"2023-01-02T14:12:16.783969Z","shell.execute_reply.started":"2023-01-02T14:12:16.765541Z","shell.execute_reply":"2023-01-02T14:12:16.782949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"# Utility to read hdf5 file\ndef read_data(file: Path):\n    with h5py.File(file, \"r\") as f:\n        file=Path(file)\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        #freq_hz = list(f[\"frequency_Hz\"])\n        ###\n        h1_stft = h1[\"SFTs\"][()]\n        ###\n        #h1_timestamp = h1[\"timestamps_GPS\"][()]\n        h1_timestamp =1\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        #l1_timestamp = l1[\"timestamps_GPS\"][()]\n        l1_timestamp =2\n        return {\n            \"H1\": [h1_stft, h1_timestamp],\n            \"L1\": [l1_stft, l1_timestamp],\n            #\"freq_hz\": freq_hz\n        }","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.818293Z","iopub.execute_input":"2023-01-02T14:12:16.818889Z","iopub.status.idle":"2023-01-02T14:12:16.827626Z","shell.execute_reply.started":"2023-01-02T14:12:16.81885Z","shell.execute_reply":"2023-01-02T14:12:16.826686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data_c(file):\n    file = Path(file)\n    with h5py.File(file, \"r\") as f:\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        freq_hz = f[\"frequency_Hz\"]\n        \n        h1_stft = h1[\"SFTs\"][()]\n        h1_timestamp = h1[\"timestamps_GPS\"][()]\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        l1_timestamp = l1[\"timestamps_GPS\"][()]\n        \n        return [h1_stft, h1_timestamp],            [l1_stft, l1_timestamp], np.array(freq_hz)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:16.948582Z","iopub.execute_input":"2023-01-02T14:12:16.949338Z","iopub.status.idle":"2023-01-02T14:12:16.957276Z","shell.execute_reply.started":"2023-01-02T14:12:16.949305Z","shell.execute_reply":"2023-01-02T14:12:16.956282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Reduced Noise Spectrograms","metadata":{}},{"cell_type":"code","source":"!pip install timm\n\nimport shutil\nimport os\nimport gc, glob, os\nfrom concurrent.futures import ProcessPoolExecutor\n\nimport h5py\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom scipy.stats import norm\nfrom timm import create_model\nfrom tqdm.notebook import tqdm\n\n\nos.makedirs('/kaggle/working/test/', exist_ok=True)\nos.makedirs('/kaggle/working/noise_reduction/test', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:12:32.025749Z","iopub.execute_input":"2023-01-02T14:12:32.027416Z","iopub.status.idle":"2023-01-02T14:12:46.465693Z","shell.execute_reply.started":"2023-01-02T14:12:32.02739Z","shell.execute_reply":"2023-01-02T14:12:46.464566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize(X):\n    X = (X[..., None].view(X.real.dtype) ** 2).sum(-1)\n    POS = int(X.size * 0.99903)\n    EXP = norm.ppf((POS + 0.4) / (X.size + 0.215))\n    scale = np.partition(X.flatten(), POS, -1)[POS]\n    X /= scale / EXP.astype(scale.dtype) ** 2\n    return X\n\ndef dataload(filepath):\n    astime = np.full([2, 360, 5760], np.nan, dtype=np.float32)\n    with h5py.File(filepath, \"r\") as f:\n        fid, _ = os.path.splitext(os.path.split(filepath)[1])\n        HT = (np.asarray(f[fid][\"H1\"][\"timestamps_GPS\"]) / 1800).round().astype(np.int64)\n        LT = (np.asarray(f[fid][\"L1\"][\"timestamps_GPS\"]) / 1800).round().astype(np.int64)\n        MIN = min(HT.min(), LT.min()); HT -= MIN; LT -= MIN\n        H1 = normalize(np.asarray(f[fid][\"H1\"][\"SFTs\"], np.complex128))\n        valid = HT < 5760; astime[0][:, HT[valid]] = H1[:, valid]\n        L1 = normalize(np.asarray(f[fid][\"L1\"][\"SFTs\"], np.complex128))\n        valid = LT < 5760; astime[1][:, LT[valid]] = L1[:, valid]\n    gc.collect()\n    return fid, astime, H1.mean(), L1.mean()\n\nclass LargeKernel_debias(nn.Conv2d):\n    def forward(self, input: torch.Tensor):\n        finput = input.flatten(0, 1)[:, None]\n        target = abs(self.weight)\n        target = target / target.sum((-1, -2), True)\n        joined_kernel = torch.cat([self.weight, target], 0)\n        reals = target.new_zeros(\n            [1, 1] + [s + p * 2 for p, s in zip(self.padding, input.shape[-2:])]\n        )\n        reals[\n            [slice(None)] * 2 + [slice(p, -p) if p != 0 else slice(None) for p in self.padding]\n        ].fill_(1)\n        output, power = torch.nn.functional.conv2d(\n            finput, joined_kernel, padding=self.padding\n        ).chunk(2, 1)\n        ratio = torch.div(*torch.nn.functional.conv2d(reals, joined_kernel).chunk(2, 1))\n        output.sub_(power.mul_(ratio))\n        output = output.unflatten(0, input.shape[:2]).flatten(1, 2)\n        #print(output.shape)\n        return output\n\ndef preprocess(num, input, H1, L1):\n    input = torch.from_numpy(input).to(\"cuda\", non_blocking=True)\n    rescale = torch.tensor([[H1, L1]]).to(\"cuda\", non_blocking=True)\n    tta = (\n        torch.randn(\n            [num, *input.shape, 2], device=input.device, dtype=torch.float32\n        )\n        .square_()\n        .sum(-1)\n    )\n    tta *= rescale[..., None, None] / 2\n    valid = ~torch.isnan(input); tta[:, valid] = input[valid].float()\n    return tta\n\ndef get_model(path):\n    model = create_model(\n        \"tf_efficientnetv2_b0\",\n        in_chans=32,\n        num_classes=2,\n    )\n    state_dict = torch.load(path)\n    C, _, H, W = state_dict[\"conv_stem.2.weight\"].shape\n    model.conv_stem = nn.Sequential(\n        nn.Identity(),\n        nn.AvgPool2d((1, 9), (1, 8), (0, 4), count_include_pad=False),\n        LargeKernel_debias(1, C, [H, W], 1, [H//2, W//2], 1, 1, False),\n        model.conv_stem,\n    )\n    model.load_state_dict(state_dict)\n    model = model.conv_stem\n    model[3] = nn.Identity()\n    \n    model = nn.Sequential(\n        model,\n        #nn.Conv3d(64, 1, (1,1))\n    )\n    \n    #print(model)\n    model.cuda().eval()\n    return model\n\n@torch.no_grad()\ndef inference(model, table, out_di, start, end):\n    FID, RES = [], []\n    output_arr = []\n    with ProcessPoolExecutor(2) as pool:\n        for fid, input, H1, L1 in pool.map(dataload, table):\n            tta = preprocess(64, input, H1, L1)\n            FID += [fid]\n            output = torch.mean(model(tta), dim=(0,1))\n            np.save('%s/%s.npy' % (out_di, fid), output.detach().cpu().numpy())\n            del(fid)\n            del(H1)\n            del(L1)\n            del(tta)\n            del(output)\n            gc.collect()\n    return FID, output_arr\n\nmodel = get_model(\"../input/g2netdetectingcontinuousgravitationalwavesv0/group0/model_best.pth\")\n\nfile_path = glob.glob(os.path.join(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/test/\", \"*.hdf5\"))\n\n# you are at part 0\n# don't forget to change the slice from [start:end]\n# to [start:] for part 10\n\n\n\n# DONT FORGET TO CHANGE PATH AND USE WHEN CHANGING TO NOISES\n\nsave_path = '/kaggle/working/consolidated/' + 'noises/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\n\ntrain = train[train[\"target\"] >= 0]\nsignals = train.loc[train['target'] == 1]\nnoises = train.loc[train['target'] == 0]\n\nuse = noises\n\nPART = 5\nNUM_PARTS = 5\n\nlength = len(use)\nnum_elems = (1 / NUM_PARTS) * length\nend = num_elems * PART\nstart = end - num_elems\n\nstart = int(start)\nend = int(end)\n\ntable = use[start:].cale_fisier\n\nfid, infer = inference(model,\n                      table,\n                      save_path,\n                      start,\n                      end)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T14:26:27.770328Z","iopub.execute_input":"2023-01-02T14:26:27.770727Z","iopub.status.idle":"2023-01-02T14:26:46.332197Z","shell.execute_reply.started":"2023-01-02T14:26:27.770694Z","shell.execute_reply":"2023-01-02T14:26:46.330332Z"},"trusted":true},"execution_count":null,"outputs":[]}]}