{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"train_only=True","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:16.295656Z","iopub.execute_input":"2023-01-02T18:42:16.296108Z","iopub.status.idle":"2023-01-02T18:42:16.300545Z","shell.execute_reply.started":"2023-01-02T18:42:16.29607Z","shell.execute_reply":"2023-01-02T18:42:16.299454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n# for filename in filenames:\n# print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-02T18:42:16.588926Z","iopub.execute_input":"2023-01-02T18:42:16.590085Z","iopub.status.idle":"2023-01-02T18:42:16.596241Z","shell.execute_reply.started":"2023-01-02T18:42:16.59002Z","shell.execute_reply":"2023-01-02T18:42:16.595061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://hal.archives-ouvertes.fr/hal-00085154/document\n\nhttps://hal.archives-ouvertes.fr/hal-00301405/document\n\nhttps://hal.archives-ouvertes.fr/hal-01714325/document","metadata":{}},{"cell_type":"code","source":"#dataset_list=['GS1','GS3','GS2','GS4','GS5','GS6','GS7','GS8','GS9','GS10','GS11','GS12','GS14','GS15']\n#datasetlist\n#GS1-gaussian-noise\n#GS2-nonstationary\n#GS3 narrowartifacts noise\n#GS4-nonstationary\n#GS5 gaussian-noise\n#GS6 gaussian-noise\n#GS7 gaussian-noise\n#GS8 -nonstationary\n#GS9 low SNR gaussian-noise\n#GS10 low SNR gaussian-noise\n#GS11 low SNR gaussian-noise\n#GS12 low SNR nonstationary\n#GS13 low SNR artifacts\n#GS14 low SNR gaussian-noise\n#List SNR short\n###Low only \"pure gaussian noise\"\n#dataset_list=['GS1','GS5','GS6','GS7','GS9','GS10','GS11']\n###High only\n# dataset_list=['GS2','GS3','GS4','GS8','GS10','GS12','GS13']\n#dataset_list=['GS1','GS3','GS2','GS4','GS5','GS6','GS7','GS8','GS9','GS10','GS11','GS12','GS13','GS14','GS15','GS16','GS17','GS18','GS19','GS20']\ndataset_list=['GS1','GS9','GS10','GS11','GS12','GS13','GS14','GS15','GS16','GS17','GS18','GS19','GS20','GS21']\n# dataset_list=['GS21']","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:25.160623Z","iopub.execute_input":"2023-01-02T18:42:25.161034Z","iopub.status.idle":"2023-01-02T18:42:25.16756Z","shell.execute_reply.started":"2023-01-02T18:42:25.160986Z","shell.execute_reply":"2023-01-02T18:42:25.166419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h5py\nfrom glob import glob\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:29.896795Z","iopub.execute_input":"2023-01-02T18:42:29.897456Z","iopub.status.idle":"2023-01-02T18:42:29.901356Z","shell.execute_reply.started":"2023-01-02T18:42:29.897424Z","shell.execute_reply":"2023-01-02T18:42:29.900196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib as mpl","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:30.096283Z","iopub.execute_input":"2023-01-02T18:42:30.096946Z","iopub.status.idle":"2023-01-02T18:42:30.101873Z","shell.execute_reply.started":"2023-01-02T18:42:30.09691Z","shell.execute_reply":"2023-01-02T18:42:30.100458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:30.310355Z","iopub.execute_input":"2023-01-02T18:42:30.311355Z","iopub.status.idle":"2023-01-02T18:42:30.316153Z","shell.execute_reply.started":"2023-01-02T18:42:30.311315Z","shell.execute_reply":"2023-01-02T18:42:30.314919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport shutil\n\n# Providing the folder path\norigin = '/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'\n#origin=Path(origin)\n# target = '../working/train/'\n# temp='../temp/'\n# if not os.path.exists(temp):\n# os.mkdir(temp)\n# if not os.path.exists(target):\n# os.mkdir(target)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:30.507561Z","iopub.execute_input":"2023-01-02T18:42:30.507975Z","iopub.status.idle":"2023-01-02T18:42:30.512624Z","shell.execute_reply.started":"2023-01-02T18:42:30.507925Z","shell.execute_reply":"2023-01-02T18:42:30.511748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor root, folders, filenames in os.walk('/kaggle/input'):\n print(root, folders)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:30.703072Z","iopub.execute_input":"2023-01-02T18:42:30.703683Z","iopub.status.idle":"2023-01-02T18:42:38.730807Z","shell.execute_reply.started":"2023-01-02T18:42:30.703647Z","shell.execute_reply":"2023-01-02T18:42:38.729916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_WIDTH=128","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.732212Z","iopub.execute_input":"2023-01-02T18:42:38.733004Z","iopub.status.idle":"2023-01-02T18:42:38.736536Z","shell.execute_reply.started":"2023-01-02T18:42:38.732972Z","shell.execute_reply":"2023-01-02T18:42:38.735726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sft_to_img(sft, ts, buckets=IMG_WIDTH):\n bucket_size = (ts.max() - ts.min()) // buckets\n idx = np.searchsorted(ts, [ts[0] + bucket_size * i for i in range(buckets)])\n # 1. ASD\n sft = np.absolute(sft) \n # 2. shrink\n global_noise_amp = np.mean(sft, axis=1)\n img = np.stack([\n np.mean(i, axis=1) if i.shape[1] > 0 else global_noise_amp for i in np.array_split(sft, idx[1:], axis=1) \n ]) \n ts_noise = img.mean(axis=1)\n # 3. Normalize\n mean, std = np.mean(img), np.std(img.astype(np.float64))\n # print(mean, std) \n\n img = img - mean\n img = img / std / 4\n img *= 128 \n img += 128\n # img = img * (255 / np.max(img))\n# img = np.clip(img, 0, 255).astype(np.uint8)\n # print(np.min(img), np.max(img), np.mean(img), np.std(img))\n# return img.T, ts_noise\n return img.T\n if DEBUG:\n img, ts_noise = sft_to_img(h1_sft, h1_ts)\n plt.figure()\n plt.imshow(img, cmap='gray')\n plt.figure()\n plt.plot(ts_noise)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.737586Z","iopub.execute_input":"2023-01-02T18:42:38.738287Z","iopub.status.idle":"2023-01-02T18:42:38.748579Z","shell.execute_reply.started":"2023-01-02T18:42:38.738253Z","shell.execute_reply":"2023-01-02T18:42:38.747475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_2d(matrix):\n # Only this is changed to use 2-norm put 2 instead of 1\n norm = np.linalg.norm(matrix, 1)\n # normalized matrix\n matrix = matrix/norm\n \n return matrix\n# Utility to read hdf5 file\ndef read_data(file: Path):\n with h5py.File(file, \"r\") as f:\n file=Path(file)\n filename = file.stem\n f = f[filename]\n h1 = f[\"H1\"]\n l1 = f[\"L1\"]\n #freq_hz = list(f[\"frequency_Hz\"])\n ###\n h1_stft = h1[\"SFTs\"][()]\n ###\n #h1_timestamp = h1[\"timestamps_GPS\"][()]\n h1_timestamp =1\n # H2 data\n l1_stft = l1[\"SFTs\"][()]\n #l1_timestamp = l1[\"timestamps_GPS\"][()]\n l1_timestamp =2\n return {\n \"H1\": [h1_stft, h1_timestamp],\n \"L1\": [l1_stft, l1_timestamp],\n #\"freq_hz\": freq_hz\n }\ndef read_data_c(file):\n file = Path(file)\n with h5py.File(file, \"r\") as f:\n filename = file.stem\n f = f[filename]\n h1 = f[\"H1\"]\n l1 = f[\"L1\"]\n freq_hz = f[\"frequency_Hz\"]\n h1_stft = np.array(f['H1']['SFTs'], dtype=np.complex128)\n# h1_stft = h1[\"SFTs\"][()]\n h1_timestamp = h1[\"timestamps_GPS\"][()]\n # H2 data\n# l1_stft = l1[\"SFTs\"][()]\n l1_stft = np.array(f['L1']['SFTs'], dtype=np.complex128)\n l1_timestamp = l1[\"timestamps_GPS\"][()]\n \n return [h1_stft, h1_timestamp], [l1_stft, l1_timestamp], np.array(freq_hz)\n\ndef power_spectrogram2(h1_sft,l1_sft):\n i=0\n \n img = np.empty((360,360,3), dtype=np.float64)\n try:\n \n a=h1_sft[:360, :4320]\n p = a.real**2 + a.imag**2\n p = normalize_2d(p)\n p /= np.mean(p) # normalize\n q=(a.real*2)*(np.angle(a))\n q=normalize_2d(q)\n# q /= np.mean(q) # normalize\n try:\n p = np.mean(p.reshape(360, 360,12), axis=2)\n q = np.mean(q.reshape(360, 360,12), axis=2)\n except:\n a=h1_sft[:360, :3960]\n p = a.real**2 + a.imag**2\n p=normalize_2d(p)\n q=(a.real*2)*(np.angle(a))\n q=normalize_2d(q)\n# p /= np.mean(p) # normalize\n# q /= np.mean(q) # normalize\n p = np.mean(p.reshape(360, 360,11), axis=2)\n q = np.mean(q.reshape(360, 360,11), axis=2)\n\n #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n\n # normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n img[:,:,0]= p*255\n# img[:,:,2]= np.ones((360,360), dtype=np.float32)\n# img[:,:,2]=img[:,:,2]*255\n\n\n img[:,:,2]= q*255\n a=l1_sft[:360, :4320]\n p = a.real**2 + a.imag**2\n p=normalize_2d(p)\n# p /= np.mean(p) # normalize\n try:\n p = np.mean(p.reshape(360, 360,12), axis=2)\n except:\n a=l1_sft[:360, :3960]\n p = a.real**2 + a.imag**2\n p=normalize_2d(p)\n# p /= np.mean(p) # normalize\n p = np.mean(p.reshape(360, 360,11), axis=2)\n\n #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n img[:,:,1]= p*255\n\n# img = np.moveaxis(img, 0, -1)\n #return np.asarray([img[0],img[1]]).astype('float32')\n except:\n print(\"empty image\")\n# img = np.moveaxis(img, 0, -1)\n\n return img \ndef plot_spectrogram(h1_sft,l1_sft,width):\n img = np.empty((360,width,3), dtype=np.float64)\n a=h1_sft\n p = a.real**2 + a.imag**2\n p=normalize_2d(p)\n q=(a.real*2)*(np.angle(a))\n q=normalize_2d(q)\n img[:,:,0]= p*255\n img[:,:,1]= p*255\n try:\n a=l1_sft\n p = a.real**2 + a.imag**2\n p=normalize_2d(p)\n img[:,:,2]= p*255\n except:\n print('bad easter egg')\n return img\n \n# plt.imshow(img)\n# plt.show()\n \ndef power_spectrogram3(h1_sft,l1_sft):\n i=0\n \n img = np.empty((360,360,3), dtype=np.float32)\n try:\n a=h1_sft[:360, :4320]*1e22\n p = a.real**2 + a.imag**2\n p /= np.mean(p) # normalize\n q=(a.real*2)*(np.angle(a))\n q /= np.mean(q) # normalize\n try:\n p = np.mean(p.reshape(360, 360,12), axis=2)\n q = np.mean(q.reshape(360, 360,12), axis=2)\n except:\n a=h1_sft[:360, :3960]*1e22\n p = a.real**2 + a.imag**2\n q=(a.real*2)*(np.angle(a))\n p /= np.mean(p) # normalize\n q /= np.mean(q) # normalize\n p = np.mean(p.reshape(360, 360,11), axis=2)\n q = np.mean(q.reshape(360, 360,11), axis=2)\n\n #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n\n # normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n img[:,:,0]= p*255\n# img[:,:,2]= np.ones((360,360), dtype=np.float32)\n# img[:,:,2]=img[:,:,2]*255\n\n\n img[:,:,2]= q*255\n a=l1_sft[:360, :4320]*1e22\n p = a.real**2 + a.imag**2\n p /= np.mean(p) # normalize\n try:\n p = np.mean(p.reshape(360, 360,12), axis=2)\n except:\n a=l1_sft[:360, :3960]*1e22\n p = a.real**2 + a.imag**2\n p /= np.mean(p) # normalize\n p = np.mean(p.reshape(360, 360,11), axis=2)\n\n #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n img[:,:,1]= p*255\n\n# img = np.moveaxis(img, 0, -1)\n #return np.asarray([img[0],img[1]]).astype('float32')\n except:\n print(\"empty image\")\n# img = np.moveaxis(img, 0, -1)\n\n return img\ndef show_power_spectrogram2(filename):\n f = h5py.File(filename, 'r')\n\n # read fourier transform coefficients\n (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n width=len(h1_sfts)\n img=power_spectrogram2(h1_sfts,l1_sfts,)\n plt.imshow(img)\n plt.show()\ndef show_power_spectrogram(filename):\n f = h5py.File(filename, 'r')\n\n # read fourier transform coefficients\n (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n width=h1_sfts.shape[1]\n img=plot_spectrogram(h1_sfts,l1_sfts,width)\n plt.rcParams[\"figure.figsize\"] = (15,5)\n plt.imshow(img)\n plt.show()\ndef test_dataset(filename):\n filename=Path(filename)\n with h5py.File(filename) as f:\n filename=filename.stem\n g = f[filename]\n print(filename)\n try:\n for ch, s in enumerate(['H1', 'L1']):\n x=g[s]['SFTs'].shape\n except:\n print('bad dataset')","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.751Z","iopub.execute_input":"2023-01-02T18:42:38.751417Z","iopub.status.idle":"2023-01-02T18:42:38.789819Z","shell.execute_reply.started":"2023-01-02T18:42:38.751388Z","shell.execute_reply":"2023-01-02T18:42:38.788497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1+1","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.792007Z","iopub.execute_input":"2023-01-02T18:42:38.792936Z","iopub.status.idle":"2023-01-02T18:42:38.801346Z","shell.execute_reply.started":"2023-01-02T18:42:38.792901Z","shell.execute_reply":"2023-01-02T18:42:38.800471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Fetching the list of all the files\n# files = os.listdir(origin)\n\n# # Fetching all the files to directory\n# for file_name in files:\n# shutil.copy(origin+file_name, target+file_name)\n# print(\"Files are copied successfully\")|","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.802616Z","iopub.execute_input":"2023-01-02T18:42:38.802894Z","iopub.status.idle":"2023-01-02T18:42:38.81059Z","shell.execute_reply.started":"2023-01-02T18:42:38.802868Z","shell.execute_reply":"2023-01-02T18:42:38.809821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"heigh=360\n# width=360\n# first_cut=4320 #360\n# compress_1=12\n# second_cut=3960 #360\n# compress_2=11\nwidth=128\nfirst_cut=4096\ncompress_1=32\nsecond_cut=4096-128\ncompress_2=31","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.812297Z","iopub.execute_input":"2023-01-02T18:42:38.812609Z","iopub.status.idle":"2023-01-02T18:42:38.820931Z","shell.execute_reply.started":"2023-01-02T18:42:38.812582Z","shell.execute_reply":"2023-01-02T18:42:38.819994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/train_labels.csv\")\nlen(train)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.822283Z","iopub.execute_input":"2023-01-02T18:42:38.822576Z","iopub.status.idle":"2023-01-02T18:42:38.842407Z","shell.execute_reply.started":"2023-01-02T18:42:38.822549Z","shell.execute_reply":"2023-01-02T18:42:38.841574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.843774Z","iopub.execute_input":"2023-01-02T18:42:38.844173Z","iopub.status.idle":"2023-01-02T18:42:38.859564Z","shell.execute_reply.started":"2023-01-02T18:42:38.84414Z","shell.execute_reply":"2023-01-02T18:42:38.858665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['cale_fisier']='/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'+train['id']+'.hdf5'","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.862967Z","iopub.execute_input":"2023-01-02T18:42:38.863569Z","iopub.status.idle":"2023-01-02T18:42:38.871502Z","shell.execute_reply.started":"2023-01-02T18:42:38.863446Z","shell.execute_reply":"2023-01-02T18:42:38.870742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train['cale_fisier']=train['target'].apply(x:'/kaggle/input/g2net-detecting-continuous-gravitational-waves/'+train['id']+'.hdf5' if x=1.0 else '/kaggle/input/g2net-detecting-continuous-gravitational-waves/'+train['id']+'.hdf5')\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.872659Z","iopub.execute_input":"2023-01-02T18:42:38.873134Z","iopub.status.idle":"2023-01-02T18:42:38.881388Z","shell.execute_reply.started":"2023-01-02T18:42:38.873105Z","shell.execute_reply":"2023-01-02T18:42:38.880484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram2(train.cale_fisier[1])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:38.882456Z","iopub.execute_input":"2023-01-02T18:42:38.883221Z","iopub.status.idle":"2023-01-02T18:42:40.278413Z","shell.execute_reply.started":"2023-01-02T18:42:38.88319Z","shell.execute_reply":"2023-01-02T18:42:40.277242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Easter eggs","metadata":{}},{"cell_type":"code","source":"easter_eggs=train[train.target<0]","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:40.280278Z","iopub.execute_input":"2023-01-02T18:42:40.281061Z","iopub.status.idle":"2023-01-02T18:42:40.289613Z","shell.execute_reply.started":"2023-01-02T18:42:40.280995Z","shell.execute_reply":"2023-01-02T18:42:40.288609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"easter_eggs=easter_eggs.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:40.291243Z","iopub.execute_input":"2023-01-02T18:42:40.292272Z","iopub.status.idle":"2023-01-02T18:42:40.298832Z","shell.execute_reply.started":"2023-01-02T18:42:40.292239Z","shell.execute_reply":"2023-01-02T18:42:40.297651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"easter_eggs.cale_fisier","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:40.301418Z","iopub.execute_input":"2023-01-02T18:42:40.302246Z","iopub.status.idle":"2023-01-02T18:42:40.312345Z","shell.execute_reply.started":"2023-01-02T18:42:40.302125Z","shell.execute_reply":"2023-01-02T18:42:40.311282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(train.cale_fisier[3])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:40.313918Z","iopub.execute_input":"2023-01-02T18:42:40.314724Z","iopub.status.idle":"2023-01-02T18:42:41.936347Z","shell.execute_reply.started":"2023-01-02T18:42:40.31468Z","shell.execute_reply":"2023-01-02T18:42:41.935428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(easter_eggs.cale_fisier[0])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:41.937761Z","iopub.execute_input":"2023-01-02T18:42:41.938779Z","iopub.status.idle":"2023-01-02T18:42:42.425452Z","shell.execute_reply.started":"2023-01-02T18:42:41.938744Z","shell.execute_reply":"2023-01-02T18:42:42.424206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(easter_eggs.cale_fisier[2])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:42.427072Z","iopub.execute_input":"2023-01-02T18:42:42.427396Z","iopub.status.idle":"2023-01-02T18:42:42.972236Z","shell.execute_reply.started":"2023-01-02T18:42:42.427368Z","shell.execute_reply":"2023-01-02T18:42:42.971239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(easter_eggs.cale_fisier[1])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:42.974145Z","iopub.execute_input":"2023-01-02T18:42:42.974525Z","iopub.status.idle":"2023-01-02T18:42:43.716584Z","shell.execute_reply.started":"2023-01-02T18:42:42.974493Z","shell.execute_reply":"2023-01-02T18:42:43.715357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram2(easter_eggs.cale_fisier[2])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:43.717834Z","iopub.execute_input":"2023-01-02T18:42:43.718192Z","iopub.status.idle":"2023-01-02T18:42:44.094411Z","shell.execute_reply.started":"2023-01-02T18:42:43.718162Z","shell.execute_reply":"2023-01-02T18:42:44.093329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.cale_fisier[555]","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:44.096012Z","iopub.execute_input":"2023-01-02T18:42:44.096679Z","iopub.status.idle":"2023-01-02T18:42:44.104193Z","shell.execute_reply.started":"2023-01-02T18:42:44.096638Z","shell.execute_reply":"2023-01-02T18:42:44.102956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS1'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = '/kaggle/input/g2netgaussian-noise-dataset/noise/'\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:44.106341Z","iopub.execute_input":"2023-01-02T18:42:44.107641Z","iopub.status.idle":"2023-01-02T18:42:45.748613Z","shell.execute_reply.started":"2023-01-02T18:42:44.107596Z","shell.execute_reply":"2023-01-02T18:42:45.747496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS1'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = '/kaggle/input/g2netgaussian-noise-dataset/signal/'\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:45.750318Z","iopub.execute_input":"2023-01-02T18:42:45.751083Z","iopub.status.idle":"2023-01-02T18:42:46.924929Z","shell.execute_reply.started":"2023-01-02T18:42:45.751019Z","shell.execute_reply":"2023-01-02T18:42:46.924162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram2(df_add.cale_fisier[1])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:46.925955Z","iopub.execute_input":"2023-01-02T18:42:46.926915Z","iopub.status.idle":"2023-01-02T18:42:48.158327Z","shell.execute_reply.started":"2023-01-02T18:42:46.926882Z","shell.execute_reply":"2023-01-02T18:42:48.157215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_dataset(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.160056Z","iopub.execute_input":"2023-01-02T18:42:48.160703Z","iopub.status.idle":"2023-01-02T18:42:48.165353Z","shell.execute_reply.started":"2023-01-02T18:42:48.160662Z","shell.execute_reply":"2023-01-02T18:42:48.164104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.167162Z","iopub.execute_input":"2023-01-02T18:42:48.16756Z","iopub.status.idle":"2023-01-02T18:42:48.179294Z","shell.execute_reply.started":"2023-01-02T18:42:48.167521Z","shell.execute_reply":"2023-01-02T18:42:48.178237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g2netnonstationary-dataset","metadata":{}},{"cell_type":"code","source":"print(os.listdir(\"../input\"))","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.187521Z","iopub.execute_input":"2023-01-02T18:42:48.187893Z","iopub.status.idle":"2023-01-02T18:42:48.19373Z","shell.execute_reply.started":"2023-01-02T18:42:48.187864Z","shell.execute_reply":"2023-01-02T18:42:48.192591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS2'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = '/kaggle/input/g2netnonstationary-dataset/signal/'\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.195347Z","iopub.execute_input":"2023-01-02T18:42:48.195739Z","iopub.status.idle":"2023-01-02T18:42:48.20408Z","shell.execute_reply.started":"2023-01-02T18:42:48.195702Z","shell.execute_reply":"2023-01-02T18:42:48.202937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS2'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/g2netnonstationary-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.206744Z","iopub.execute_input":"2023-01-02T18:42:48.207111Z","iopub.status.idle":"2023-01-02T18:42:48.218137Z","shell.execute_reply.started":"2023-01-02T18:42:48.207081Z","shell.execute_reply":"2023-01-02T18:42:48.217084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### g2net-datasetnarrowartifacts","metadata":{}},{"cell_type":"code","source":"dataset_number='GS3'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = '/kaggle/input/g2net-datasetnarrowartifacts/noise/'\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.219327Z","iopub.execute_input":"2023-01-02T18:42:48.219611Z","iopub.status.idle":"2023-01-02T18:42:48.228749Z","shell.execute_reply.started":"2023-01-02T18:42:48.219584Z","shell.execute_reply":"2023-01-02T18:42:48.227906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### addon-g2netnonstationary-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS4'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/addon-g2netnonstationary-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.229619Z","iopub.execute_input":"2023-01-02T18:42:48.229934Z","iopub.status.idle":"2023-01-02T18:42:48.238821Z","shell.execute_reply.started":"2023-01-02T18:42:48.229891Z","shell.execute_reply":"2023-01-02T18:42:48.237858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS4'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/addon-g2netnonstationary-dataset/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.239851Z","iopub.execute_input":"2023-01-02T18:42:48.240433Z","iopub.status.idle":"2023-01-02T18:42:48.251884Z","shell.execute_reply.started":"2023-01-02T18:42:48.24039Z","shell.execute_reply":"2023-01-02T18:42:48.251099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### addon-g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS5'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/addon-g2netgaussian-noise-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.252792Z","iopub.execute_input":"2023-01-02T18:42:48.253107Z","iopub.status.idle":"2023-01-02T18:42:48.270289Z","shell.execute_reply.started":"2023-01-02T18:42:48.253078Z","shell.execute_reply":"2023-01-02T18:42:48.269263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS5'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/addon-g2netgaussian-noise-dataset/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.271945Z","iopub.execute_input":"2023-01-02T18:42:48.272474Z","iopub.status.idle":"2023-01-02T18:42:48.28107Z","shell.execute_reply.started":"2023-01-02T18:42:48.272368Z","shell.execute_reply":"2023-01-02T18:42:48.279859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### add-on-narrow-instrumental-artifacts","metadata":{}},{"cell_type":"code","source":"dataset_number='GS5'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/add-on-narrow-instrumental-artifacts-g2net-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.282303Z","iopub.execute_input":"2023-01-02T18:42:48.282651Z","iopub.status.idle":"2023-01-02T18:42:48.291357Z","shell.execute_reply.started":"2023-01-02T18:42:48.282619Z","shell.execute_reply":"2023-01-02T18:42:48.290299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2nd-addon-g2netgaussian-noise-dataset\n\n","metadata":{}},{"cell_type":"code","source":"dataset_number='GS6'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/2nd-addon-g2netgaussian-noise-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.292786Z","iopub.execute_input":"2023-01-02T18:42:48.293103Z","iopub.status.idle":"2023-01-02T18:42:48.301959Z","shell.execute_reply.started":"2023-01-02T18:42:48.293076Z","shell.execute_reply":"2023-01-02T18:42:48.301081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS6'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/2nd-addon-g2netgaussian-noise-dataset/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.302996Z","iopub.execute_input":"2023-01-02T18:42:48.304288Z","iopub.status.idle":"2023-01-02T18:42:48.31597Z","shell.execute_reply.started":"2023-01-02T18:42:48.304243Z","shell.execute_reply":"2023-01-02T18:42:48.31508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3rdaddon-g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS7'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/3rdaddon-g2netgaussian-noise-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.317384Z","iopub.execute_input":"2023-01-02T18:42:48.31774Z","iopub.status.idle":"2023-01-02T18:42:48.325943Z","shell.execute_reply.started":"2023-01-02T18:42:48.317709Z","shell.execute_reply":"2023-01-02T18:42:48.325101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS7'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/3rdaddon-g2netgaussian-noise-dataset/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.326893Z","iopub.execute_input":"2023-01-02T18:42:48.327567Z","iopub.status.idle":"2023-01-02T18:42:48.33721Z","shell.execute_reply.started":"2023-01-02T18:42:48.327536Z","shell.execute_reply":"2023-01-02T18:42:48.336321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ND-ADDon G2Net-Non-stationary Dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS8'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/ndaddon-g2netnonstationary-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.338464Z","iopub.execute_input":"2023-01-02T18:42:48.340234Z","iopub.status.idle":"2023-01-02T18:42:48.348761Z","shell.execute_reply.started":"2023-01-02T18:42:48.340168Z","shell.execute_reply":"2023-01-02T18:42:48.347885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS8'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/ndaddon-g2netnonstationary-dataset/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])\n ","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.350224Z","iopub.execute_input":"2023-01-02T18:42:48.350673Z","iopub.status.idle":"2023-01-02T18:42:48.361727Z","shell.execute_reply.started":"2023-01-02T18:42:48.350632Z","shell.execute_reply":"2023-01-02T18:42:48.360805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LowSNRG2Net-Gaussian Noise Dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS9'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/lowsnrg2net-gaussian-noise-dataset/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:48.362894Z","iopub.execute_input":"2023-01-02T18:42:48.363711Z","iopub.status.idle":"2023-01-02T18:42:49.957807Z","shell.execute_reply.started":"2023-01-02T18:42:48.36368Z","shell.execute_reply":"2023-01-02T18:42:49.956799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS9'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/lowsnrg2net-gaussian-noise-dataset/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:49.959147Z","iopub.execute_input":"2023-01-02T18:42:49.959654Z","iopub.status.idle":"2023-01-02T18:42:51.428421Z","shell.execute_reply.started":"2023-01-02T18:42:49.959623Z","shell.execute_reply":"2023-01-02T18:42:51.427378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GeneratinglowSNRgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS10'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/generatinglowsnrgravitywaves/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:51.429889Z","iopub.execute_input":"2023-01-02T18:42:51.430259Z","iopub.status.idle":"2023-01-02T18:42:51.443204Z","shell.execute_reply.started":"2023-01-02T18:42:51.430227Z","shell.execute_reply":"2023-01-02T18:42:51.441819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS10'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/generatinglowsnrgravitywaves/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:51.444679Z","iopub.execute_input":"2023-01-02T18:42:51.445052Z","iopub.status.idle":"2023-01-02T18:42:52.940599Z","shell.execute_reply.started":"2023-01-02T18:42:51.44498Z","shell.execute_reply":"2023-01-02T18:42:52.939466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2ndgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS11'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/2ndgeneratinglowsnrgravitywaves/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:52.941728Z","iopub.execute_input":"2023-01-02T18:42:52.94202Z","iopub.status.idle":"2023-01-02T18:42:52.956959Z","shell.execute_reply.started":"2023-01-02T18:42:52.941993Z","shell.execute_reply":"2023-01-02T18:42:52.956055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS11'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/2ndgeneratinglowsnrgravitywaves/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:52.958294Z","iopub.execute_input":"2023-01-02T18:42:52.958584Z","iopub.status.idle":"2023-01-02T18:42:54.307784Z","shell.execute_reply.started":"2023-01-02T18:42:52.958556Z","shell.execute_reply":"2023-01-02T18:42:54.306657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### generatinglowsnrnonstationarygravitywaves\n\n","metadata":{}},{"cell_type":"code","source":"dataset_number='GS12'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/generatinglowsnrnonstationarygravitywaves/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:54.309208Z","iopub.execute_input":"2023-01-02T18:42:54.309581Z","iopub.status.idle":"2023-01-02T18:42:54.321868Z","shell.execute_reply.started":"2023-01-02T18:42:54.30955Z","shell.execute_reply":"2023-01-02T18:42:54.320817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS12'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/generatinglowsnrnonstationarygravitywaves/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:54.323459Z","iopub.execute_input":"2023-01-02T18:42:54.323794Z","iopub.status.idle":"2023-01-02T18:42:55.603014Z","shell.execute_reply.started":"2023-01-02T18:42:54.323746Z","shell.execute_reply":"2023-01-02T18:42:55.602213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### set3-generating-low-snr-nonstationary","metadata":{}},{"cell_type":"code","source":"dataset_number='GS13'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/set3-generating-low-snr-nonstationary/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:55.604265Z","iopub.execute_input":"2023-01-02T18:42:55.60486Z","iopub.status.idle":"2023-01-02T18:42:55.626229Z","shell.execute_reply.started":"2023-01-02T18:42:55.604827Z","shell.execute_reply":"2023-01-02T18:42:55.625098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS13'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/set3-generating-low-snr-nonstationary/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:55.627592Z","iopub.execute_input":"2023-01-02T18:42:55.627943Z","iopub.status.idle":"2023-01-02T18:42:57.405168Z","shell.execute_reply.started":"2023-01-02T18:42:55.627911Z","shell.execute_reply":"2023-01-02T18:42:57.404102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### snrlowgeavitywavessimulation","metadata":{}},{"cell_type":"code","source":"dataset_number='GS14'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/snrlowgeavitywavessimulation/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:57.406889Z","iopub.execute_input":"2023-01-02T18:42:57.40733Z","iopub.status.idle":"2023-01-02T18:42:57.423604Z","shell.execute_reply.started":"2023-01-02T18:42:57.407289Z","shell.execute_reply":"2023-01-02T18:42:57.422453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS14'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/snrlowgeavitywavessimulation/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:57.425339Z","iopub.execute_input":"2023-01-02T18:42:57.426086Z","iopub.status.idle":"2023-01-02T18:42:58.908996Z","shell.execute_reply.started":"2023-01-02T18:42:57.426037Z","shell.execute_reply":"2023-01-02T18:42:58.90796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## lowsnrnonstationary","metadata":{}},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/lowsnrnonstationary/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:58.918244Z","iopub.execute_input":"2023-01-02T18:42:58.918942Z","iopub.status.idle":"2023-01-02T18:42:58.931426Z","shell.execute_reply.started":"2023-01-02T18:42:58.918904Z","shell.execute_reply":"2023-01-02T18:42:58.930063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/lowsnrnonstationary/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:42:58.932819Z","iopub.execute_input":"2023-01-02T18:42:58.933483Z","iopub.status.idle":"2023-01-02T18:43:00.294159Z","shell.execute_reply.started":"2023-01-02T18:42:58.933434Z","shell.execute_reply":"2023-01-02T18:43:00.293069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### artifactsgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/artifactsgeneratinglowsnrgravitywaves/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:00.295414Z","iopub.execute_input":"2023-01-02T18:43:00.295753Z","iopub.status.idle":"2023-01-02T18:43:00.309254Z","shell.execute_reply.started":"2023-01-02T18:43:00.295724Z","shell.execute_reply":"2023-01-02T18:43:00.307959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/artifactsgeneratinglowsnrgravitywaves/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:00.310674Z","iopub.execute_input":"2023-01-02T18:43:00.311115Z","iopub.status.idle":"2023-01-02T18:43:01.534869Z","shell.execute_reply.started":"2023-01-02T18:43:00.311071Z","shell.execute_reply":"2023-01-02T18:43:01.53408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2ndartifactsgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS16'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/2ndartifactsgeneratinglowsnrgravitywaves/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:01.536232Z","iopub.execute_input":"2023-01-02T18:43:01.536772Z","iopub.status.idle":"2023-01-02T18:43:01.549879Z","shell.execute_reply.started":"2023-01-02T18:43:01.536739Z","shell.execute_reply":"2023-01-02T18:43:01.548603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS16'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/2ndartifactsgeneratinglowsnrgravitywaves/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:01.551465Z","iopub.execute_input":"2023-01-02T18:43:01.552005Z","iopub.status.idle":"2023-01-02T18:43:03.040949Z","shell.execute_reply.started":"2023-01-02T18:43:01.551974Z","shell.execute_reply":"2023-01-02T18:43:03.039957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### trimmedg2dataset1","metadata":{}},{"cell_type":"code","source":"dataset_number='GS17'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/trimmedg2dataset1/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:03.042548Z","iopub.execute_input":"2023-01-02T18:43:03.04285Z","iopub.status.idle":"2023-01-02T18:43:03.057589Z","shell.execute_reply.started":"2023-01-02T18:43:03.042821Z","shell.execute_reply":"2023-01-02T18:43:03.056297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS17'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/trimmedg2dataset1/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:03.059088Z","iopub.execute_input":"2023-01-02T18:43:03.059527Z","iopub.status.idle":"2023-01-02T18:43:04.478732Z","shell.execute_reply.started":"2023-01-02T18:43:03.059485Z","shell.execute_reply":"2023-01-02T18:43:04.477611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3ndartifactsgeneratinglowsnrgravit","metadata":{}},{"cell_type":"code","source":"dataset_number='GS18'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/3ndartifactsgeneratinglowsnrgravit/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:04.480314Z","iopub.execute_input":"2023-01-02T18:43:04.480873Z","iopub.status.idle":"2023-01-02T18:43:04.495159Z","shell.execute_reply.started":"2023-01-02T18:43:04.480837Z","shell.execute_reply":"2023-01-02T18:43:04.494056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS18'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/3ndartifactsgeneratinglowsnrgravit/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:04.497155Z","iopub.execute_input":"2023-01-02T18:43:04.497567Z","iopub.status.idle":"2023-01-02T18:43:05.971874Z","shell.execute_reply.started":"2023-01-02T18:43:04.497527Z","shell.execute_reply":"2023-01-02T18:43:05.970836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### trimmed2generatinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS19'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/trimmed2generatinglowsnrgravitywaves/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:05.973164Z","iopub.execute_input":"2023-01-02T18:43:05.973478Z","iopub.status.idle":"2023-01-02T18:43:05.98768Z","shell.execute_reply.started":"2023-01-02T18:43:05.973448Z","shell.execute_reply":"2023-01-02T18:43:05.986458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS19'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/trimmed2generatinglowsnrgravitywaves/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:05.989192Z","iopub.execute_input":"2023-01-02T18:43:05.989823Z","iopub.status.idle":"2023-01-02T18:43:07.227243Z","shell.execute_reply.started":"2023-01-02T18:43:05.989771Z","shell.execute_reply":"2023-01-02T18:43:07.226068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### set4generatinglowsnrnonstationary","metadata":{}},{"cell_type":"code","source":"dataset_number='GS20'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/set4generatinglowsnrnonstationary/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:07.228937Z","iopub.execute_input":"2023-01-02T18:43:07.229695Z","iopub.status.idle":"2023-01-02T18:43:07.241764Z","shell.execute_reply.started":"2023-01-02T18:43:07.229661Z","shell.execute_reply":"2023-01-02T18:43:07.240729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"dataset_number='GS20'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/set4generatinglowsnrnonstationary/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['cale_fisier']=add_on+df_add.id\n train=pd.concat([train, df_add])\n train=train.reset_index(drop=True)\n show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:07.243275Z","iopub.execute_input":"2023-01-02T18:43:07.243583Z","iopub.status.idle":"2023-01-02T18:43:08.642008Z","shell.execute_reply.started":"2023-01-02T18:43:07.243554Z","shell.execute_reply":"2023-01-02T18:43:08.640832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')\ntrain=train[train['target'] >=0]\ntrain=train.reset_index()\ntrain = train.sample(frac=1)\ntrain=train.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:08.64348Z","iopub.execute_input":"2023-01-02T18:43:08.643819Z","iopub.status.idle":"2023-01-02T18:43:08.666526Z","shell.execute_reply.started":"2023-01-02T18:43:08.643787Z","shell.execute_reply":"2023-01-02T18:43:08.664745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:08.668Z","iopub.execute_input":"2023-01-02T18:43:08.668395Z","iopub.status.idle":"2023-01-02T18:43:08.680327Z","shell.execute_reply.started":"2023-01-02T18:43:08.668363Z","shell.execute_reply":"2023-01-02T18:43:08.679477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['filetype']=0","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:08.685083Z","iopub.execute_input":"2023-01-02T18:43:08.685771Z","iopub.status.idle":"2023-01-02T18:43:08.691196Z","shell.execute_reply.started":"2023-01-02T18:43:08.68572Z","shell.execute_reply":"2023-01-02T18:43:08.689955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.cale_fisier[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:08.692797Z","iopub.execute_input":"2023-01-02T18:43:08.693165Z","iopub.status.idle":"2023-01-02T18:43:08.702879Z","shell.execute_reply.started":"2023-01-02T18:43:08.693134Z","shell.execute_reply":"2023-01-02T18:43:08.70208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:08.704328Z","iopub.execute_input":"2023-01-02T18:43:08.704874Z","iopub.status.idle":"2023-01-02T18:43:08.713832Z","shell.execute_reply.started":"2023-01-02T18:43:08.704843Z","shell.execute_reply":"2023-01-02T18:43:08.712981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndataset_number='GS21'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/sinteticsignals/noise/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=0\n df_add['filetype']=1\n df_add['cale_fisier']=add_on+df_add.id","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:08.715546Z","iopub.execute_input":"2023-01-02T18:43:08.716224Z","iopub.status.idle":"2023-01-02T18:43:08.730419Z","shell.execute_reply.started":"2023-01-02T18:43:08.716184Z","shell.execute_reply":"2023-01-02T18:43:08.728878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:45:22.593923Z","iopub.execute_input":"2023-01-02T18:45:22.594375Z","iopub.status.idle":"2023-01-02T18:45:22.611353Z","shell.execute_reply.started":"2023-01-02T18:45:22.594339Z","shell.execute_reply":"2023-01-02T18:45:22.610132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:30.536044Z","iopub.execute_input":"2023-01-02T18:43:30.53668Z","iopub.status.idle":"2023-01-02T18:43:30.548042Z","shell.execute_reply.started":"2023-01-02T18:43:30.536642Z","shell.execute_reply":"2023-01-02T18:43:30.54707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndataset_number='GS21'\nif dataset_number in dataset_list:\n print('ok:',dataset_number)\n add_on = \"/kaggle/input/sinteticsignals/signal/\"\n files_add_on = os.listdir(add_on)\n df_add=pd.DataFrame() \n df_add=pd.DataFrame(files_add_on)\n df_add=df_add.rename(columns={0: \"id\"})\n df_add['target']=1\n df_add['filetype']=1\n df_add['cale_fisier']=add_on+df_add.id","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:34.771701Z","iopub.execute_input":"2023-01-02T18:43:34.772085Z","iopub.status.idle":"2023-01-02T18:43:34.783911Z","shell.execute_reply.started":"2023-01-02T18:43:34.772054Z","shell.execute_reply":"2023-01-02T18:43:34.782762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_add.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:43:40.596233Z","iopub.execute_input":"2023-01-02T18:43:40.59662Z","iopub.status.idle":"2023-01-02T18:43:40.608408Z","shell.execute_reply.started":"2023-01-02T18:43:40.596588Z","shell.execute_reply":"2023-01-02T18:43:40.607355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.concat([train, df_add])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:44:00.715291Z","iopub.execute_input":"2023-01-02T18:44:00.716058Z","iopub.status.idle":"2023-01-02T18:44:00.726317Z","shell.execute_reply.started":"2023-01-02T18:44:00.716008Z","shell.execute_reply":"2023-01-02T18:44:00.725332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train= train.sample(frac=1)\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:45:02.708335Z","iopub.execute_input":"2023-01-02T18:45:02.708757Z","iopub.status.idle":"2023-01-02T18:45:02.718391Z","shell.execute_reply.started":"2023-01-02T18:45:02.708724Z","shell.execute_reply":"2023-01-02T18:45:02.717297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# short_train_testing=train.sample(100)\n# short_train_testing=short_train_testing.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:08:19.781011Z","iopub.status.idle":"2023-01-02T18:08:19.781595Z","shell.execute_reply.started":"2023-01-02T18:08:19.781302Z","shell.execute_reply":"2023-01-02T18:08:19.781325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 5\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:45:47.194131Z","iopub.execute_input":"2023-01-02T18:45:47.194511Z","iopub.status.idle":"2023-01-02T18:45:47.200081Z","shell.execute_reply.started":"2023-01-02T18:45:47.19448Z","shell.execute_reply":"2023-01-02T18:45:47.198974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:45:50.321135Z","iopub.execute_input":"2023-01-02T18:45:50.321859Z","iopub.status.idle":"2023-01-02T18:45:55.084559Z","shell.execute_reply.started":"2023-01-02T18:45:50.321823Z","shell.execute_reply":"2023-01-02T18:45:55.083423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:07.986994Z","iopub.execute_input":"2023-01-02T18:46:07.987419Z","iopub.status.idle":"2023-01-02T18:46:08.590834Z","shell.execute_reply.started":"2023-01-02T18:46:07.987384Z","shell.execute_reply":"2023-01-02T18:46:08.589933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = StratifiedKFold(n_splits=FOLDS, random_state=SEED, shuffle=True)\nfor i, (train_idx, valid_idx) in enumerate(kfold.split(train, train.target)): \n train.loc[valid_idx, 'fold'] = int(i)\nfolds = train \ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:10.934944Z","iopub.execute_input":"2023-01-02T18:46:10.93535Z","iopub.status.idle":"2023-01-02T18:46:10.962493Z","shell.execute_reply.started":"2023-01-02T18:46:10.93532Z","shell.execute_reply":"2023-01-02T18:46:10.961469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n if isinstance(value, type(tf.constant(0))):\n value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\ndef _float_feature(value):\n \"\"\"Returns a float_list from a float / double.\"\"\"\n return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\ndef _int64_feature(value):\n \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:14.743766Z","iopub.execute_input":"2023-01-02T18:46:14.744927Z","iopub.status.idle":"2023-01-02T18:46:14.753476Z","shell.execute_reply.started":"2023-01-02T18:46:14.744879Z","shell.execute_reply":"2023-01-02T18:46:14.75218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"serialize test and train","metadata":{}},{"cell_type":"code","source":"def serialize_example_train(img, tgt, name):\n feature = {\n 'spectrogram': _bytes_feature(img),\n 'target': _float_feature(tgt),\n 'id': _bytes_feature(name),\n }\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n return example_proto.SerializeToString()\n\ndef serialize_example_test(img, name):\n feature = {\n 'spectrogram': _bytes_feature(img),\n 'id': _bytes_feature(name),\n }\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:17.026245Z","iopub.execute_input":"2023-01-02T18:46:17.026715Z","iopub.status.idle":"2023-01-02T18:46:17.034099Z","shell.execute_reply.started":"2023-01-02T18:46:17.026664Z","shell.execute_reply":"2023-01-02T18:46:17.032953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler(feature_range=(0, 255))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:17.929481Z","iopub.execute_input":"2023-01-02T18:46:17.930206Z","iopub.status.idle":"2023-01-02T18:46:17.935849Z","shell.execute_reply.started":"2023-01-02T18:46:17.930157Z","shell.execute_reply":"2023-01-02T18:46:17.934693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import cv2","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-01T17:55:29.125993Z","iopub.execute_input":"2023-01-01T17:55:29.12638Z","iopub.status.idle":"2023-01-01T17:55:29.131252Z","shell.execute_reply.started":"2023-01-01T17:55:29.126351Z","shell.execute_reply":"2023-01-01T17:55:29.130112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img = cv2.imread('blabla.jpg')\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T17:55:29.62355Z","iopub.execute_input":"2023-01-01T17:55:29.624541Z","iopub.status.idle":"2023-01-01T17:55:29.628015Z","shell.execute_reply.started":"2023-01-01T17:55:29.624506Z","shell.execute_reply":"2023-01-01T17:55:29.627289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# b,g,r = cv2.split(img)\n# k = np.zeros_like(b)\n# # b = cv2.merge([b,k,k])\n# # g = cv2.merge([k,g,k])\n# # r = cv2.merge([k,k,r])\n# #Show the image with matplotlib\n# plt.imshow(b)\n# plt.show()\n# plt.imshow(g)\n# plt.show()\n# plt.imshow(r)\n# plt.show()\n# # plt.imshow(\"red\",r)\n# # cv2.imshow(\"green\",g)\n# # cv2.imshow(\"blue\",b)\n# # cv2.waitKey(0)\n# # cv2.destroyAllWindows()","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-01T17:55:30.223615Z","iopub.execute_input":"2023-01-01T17:55:30.224025Z","iopub.status.idle":"2023-01-01T17:55:30.229539Z","shell.execute_reply.started":"2023-01-01T17:55:30.223994Z","shell.execute_reply":"2023-01-01T17:55:30.228329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in range(FOLDS):\n ct = (folds['fold'] == f).sum()\n idx = folds[folds['fold'] == f].index\n\n print(ct)\n print('Writing TFRecord %i of %i...'%(f,ct))\n \n with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(f,ct)) as writer:\n for _, i in enumerate((tqdm(idx))):\n r = folds.iloc[i]\n y = np.float32(r.target)\n file_id = r.id\n filename=r.cale_fisier\n if r.filetype==0:\n img = np.empty((360, width, 2), dtype=np.float64)\n (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n imagine=sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)\n img[...,0] = imagine\n imagine=sft_to_img(l1_sfts, l1_ts, buckets=IMG_WIDTH)\n img[...,1] = imagine\n else:\n img=np.load(filename)\n img=img.astype('float64')\n # filename = '%s/train/%s.hdf5' % (di, file_id)\n # filename=\n # try:\n # with h5py.File(filename, 'r') as f:\n # g = f[file_id]\n # print(file_id)\n # for ch, s in enumerate(['H1', 'L1']):\n # x=g[s]['SFTs'].shape\n # print('shape x=',x)\n # try:\n # a = np.array(g[s]['SFTs'], dtype=np.complex128)\n # a = a[:360, :first_cut] # Fourier coefficient complex64\n # p = 2*(a.real**2 + a.imag**2) # power\n # p = normalize_2d(p)\n # p = np.mean(p.reshape(360, width,compress_1), axis=2)\n # except:\n # a = np.array(g[s]['SFTs'], dtype=np.complex128)\n # a = a[:360, :second_cut] # Fourier coefficient complex64\n # p = 2*(a.real**2 + a.imag**2) # power\n # p = normalize_2d(p)\n # p = np.mean(p.reshape(360,width,compress_2), axis=2)\n # img[...,ch] = p\n # # try:\n # # a=g[\"H1\"]\n # # a=a['SFTs'][:360, :4320]*1e22\n\n\n # # #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n # # q = np.sin(np.angle(a))*(np.abs(a)**2) # power\n # # q /= np.mean(q) # normalize\n # # q = np.mean(q.reshape(360, 360,12), axis=2)\n # # print(np.min(q),np.max(q))\n # # scaler = MinMaxScaler(feature_range=(0, 255))\n # # scaler = scaler.fit(q)\n # # img[...,2] = q\n # # #cv2.imwrite(\"blabla.jpg\",img)\n # # except:\n # # a=g[\"H1\"]\n # # a=a['SFTs'][:360, :4320]*1e22\n # # p = np.sin(np.angle(a))*(np.abs(a)**2) # power\n # # p /= np.mean(p) # normalize\n # # p = np.mean(p.reshape(360, 360,11), axis=2)\n # # scaler = scaler.fit(p)\n # # img[...,2] = p\n # except:\n # print('eroare')\n \n serialized_img = tf.io.serialize_tensor(img)\n example = serialize_example_train(serialized_img, y, str.encode(file_id))\n writer.write(example)\n ","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:47:04.793852Z","iopub.execute_input":"2023-01-02T18:47:04.794287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T15:32:43.359602Z","iopub.execute_input":"2022-12-18T15:32:43.361006Z","iopub.status.idle":"2022-12-18T15:32:43.380659Z","shell.execute_reply.started":"2022-12-18T15:32:43.360958Z","shell.execute_reply":"2022-12-18T15:32:43.37958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T15:32:43.382347Z","iopub.execute_input":"2022-12-18T15:32:43.382704Z","iopub.status.idle":"2022-12-18T15:32:43.393141Z","shell.execute_reply.started":"2022-12-18T15:32:43.382673Z","shell.execute_reply":"2022-12-18T15:32:43.391983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TEST TF record","metadata":{}},{"cell_type":"code","source":"test_df=pd.DataFrame()\ntest_df = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\n#test_files = glob(f\"{ROOT_DIR}/test/*.hdf5\")\ntest_df['cale_fisier']='/kaggle/input/g2net-detecting-continuous-gravitational-waves/test/'+test_df['id']+'.hdf5'\n# test_df=test_df.sample(10)\n# test_df=test_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:09:49.652204Z","iopub.execute_input":"2023-01-01T17:09:49.652811Z","iopub.status.idle":"2023-01-01T17:09:49.684778Z","shell.execute_reply.started":"2023-01-01T17:09:49.652778Z","shell.execute_reply":"2023-01-01T17:09:49.683978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:09:50.476911Z","iopub.execute_input":"2023-01-01T17:09:50.477477Z","iopub.status.idle":"2023-01-01T17:09:50.495907Z","shell.execute_reply.started":"2023-01-01T17:09:50.477442Z","shell.execute_reply":"2023-01-01T17:09:50.494947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df=test_df.sample(10)\n# test_df=test_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:32:49.606077Z","iopub.execute_input":"2023-01-01T17:32:49.606446Z","iopub.status.idle":"2023-01-01T17:32:49.614008Z","shell.execute_reply.started":"2023-01-01T17:32:49.60642Z","shell.execute_reply":"2023-01-01T17:32:49.612721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:22:43.54192Z","iopub.execute_input":"2023-01-01T18:22:43.542278Z","iopub.status.idle":"2023-01-01T18:22:43.548408Z","shell.execute_reply.started":"2023-01-01T18:22:43.542241Z","shell.execute_reply":"2023-01-01T18:22:43.547348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df = pd.read_csv(di + '/sample_submission.csv')\ntrain_only=False\nif train_only:\n print('skip the nonstationary model dataset save')\nelse:\n ct = len(test_df)\n idx = test_df.index\n f = 0\n print(ct)\n print('Writing TFRecord %i of %i...'%(f,ct))\n xi=0\n with tf.io.TFRecordWriter('test%.2i-%i.tfrec'%(f,ct)) as writer:\n for _, i in enumerate((tqdm(idx))):\n r = test_df.iloc[i]\n file_id = r.id\n filename=r.cale_fisier\n img = np.empty((360, width, 2), dtype=np.float64)\n (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n# DEBUG=True\n #sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)\n imagine=sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)\n img[...,0] = imagine\n imagine=sft_to_img(l1_sfts, l1_ts, buckets=IMG_WIDTH)\n img[...,1] = imagine\n #filename = '%s/test/%s.hdf5' % (di, file_id)\n# with h5py.File(filename, 'r') as f:\n# g = f[file_id]\n# #print('unu')\n# for ch, s in enumerate(['H1', 'L1']):\n# print(xi)\n# xi=xi+1\n \n# try:\n# a = np.array(g[s]['SFTs'], dtype=np.complex128)\n# a = a[:360, :first_cut] # Fourier coefficient complex64\n# p = 2*(a.real**2 + a.imag**2) # power\n# p = normalize_2d(p)\n# p = np.mean(p.reshape(360, width,compress_1), axis=2)\n# except:\n# a = np.array(g[s]['SFTs'], dtype=np.complex128)\n# a = a[:360, :second_cut] # Fourier coefficient complex64\n# p = 2*(a.real**2 + a.imag**2) # power\n# p = normalize_2d(p)\n# p = np.mean(p.reshape(360,width,compress_2), axis=2)\n# img[...,ch] = p\n\n# # try:\n# # a=g[\"H1\"]\n# # a=a['SFTs'][:360, :4320]*1e22\n# # #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n# # q = np.sin(np.angle(a))*(np.abs(a)**2) # power\n# # q /= np.mean(q) # normalize\n# # q = np.mean(q.reshape(360, 360,12), axis=2)\n# # print(np.min(q),np.max(q))\n# # scaler = MinMaxScaler(feature_range=(0, 255))\n# # scaler = scaler.fit(q)\n# # img[...,2] = q\n# # #cv2.imwrite(\"blabla.jpg\",img)\n# # except:\n# # a=g[\"H1\"]\n# # a=a['SFTs'][:360, :3960]*1e22\n# # p = np.sin(np.angle(a))*(np.abs(a)**2) # power\n# # p /= np.mean(p) # normalize\n# # p = np.mean(p.reshape(360, 360,11), axis=2)\n# # scaler = scaler.fit(p)\n# # img[...,2] = p \n serialized_img = tf.io.serialize_tensor(img)\n example = serialize_example_test(serialized_img, str.encode(file_id))\n writer.write(example)\ntrain_only=False","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:26:59.286113Z","iopub.execute_input":"2023-01-01T18:26:59.286475Z","iopub.status.idle":"2023-01-01T18:27:06.486334Z","shell.execute_reply.started":"2023-01-01T18:26:59.286445Z","shell.execute_reply":"2023-01-01T18:27:06.485329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So simple to get TFREC :)\nThank you!!ITK8191 !","metadata":{}},{"cell_type":"markdown","source":"Load splitted data","metadata":{}},{"cell_type":"markdown","source":"Make a directory","metadata":{}},{"cell_type":"code","source":"#filter_df=pd.read_csv(\"/kaggle/input/flipacoinoncemodel-not-a-eda/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:22.106396Z","iopub.execute_input":"2023-01-01T17:53:22.10706Z","iopub.status.idle":"2023-01-01T17:53:22.111699Z","shell.execute_reply.started":"2023-01-01T17:53:22.107022Z","shell.execute_reply":"2023-01-01T17:53:22.110722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df = pd.merge(test_df,filter_df, on=['id','id'])","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:22.783018Z","iopub.execute_input":"2023-01-01T17:53:22.783409Z","iopub.status.idle":"2023-01-01T17:53:22.787565Z","shell.execute_reply.started":"2023-01-01T17:53:22.783376Z","shell.execute_reply":"2023-01-01T17:53:22.786684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df=test_df.rename(columns={\"target_y\": \"target\"})\n# test_df=test_df.drop(['target_x'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:23.096975Z","iopub.execute_input":"2023-01-01T17:53:23.097954Z","iopub.status.idle":"2023-01-01T17:53:23.103065Z","shell.execute_reply.started":"2023-01-01T17:53:23.09792Z","shell.execute_reply":"2023-01-01T17:53:23.101006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imagine=sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:58:33.966974Z","iopub.execute_input":"2023-01-01T17:58:33.967579Z","iopub.status.idle":"2023-01-01T17:58:34.010877Z","shell.execute_reply.started":"2023-01-01T17:58:33.96754Z","shell.execute_reply":"2023-01-01T17:58:34.009495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# img, ts_noise = sft_to_img(h1_sfts, h1_ts)\nimg = sft_to_img(h1_sfts, h1_ts)\nplt.figure()\nplt.imshow(img, cmap='viridis')\n","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:24:35.322671Z","iopub.execute_input":"2023-01-01T18:24:35.323155Z","iopub.status.idle":"2023-01-01T18:24:35.488611Z","shell.execute_reply.started":"2023-01-01T18:24:35.323116Z","shell.execute_reply":"2023-01-01T18:24:35.487191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imagine","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:58:48.161469Z","iopub.execute_input":"2023-01-01T17:58:48.161868Z","iopub.status.idle":"2023-01-01T17:58:48.171917Z","shell.execute_reply.started":"2023-01-01T17:58:48.161835Z","shell.execute_reply":"2023-01-01T17:58:48.170635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:23.726417Z","iopub.execute_input":"2023-01-01T17:53:23.727108Z","iopub.status.idle":"2023-01-01T17:53:23.731963Z","shell.execute_reply.started":"2023-01-01T17:53:23.727072Z","shell.execute_reply":"2023-01-01T17:53:23.730722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# sns.displot(data=test_df, x=\"target\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:24.443878Z","iopub.execute_input":"2023-01-01T17:53:24.444231Z","iopub.status.idle":"2023-01-01T17:53:24.447984Z","shell.execute_reply.started":"2023-01-01T17:53:24.444202Z","shell.execute_reply":"2023-01-01T17:53:24.447104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# low_score=test_df[test_df[\"target\"]<=0.68]\n# low_score=low_score.reset_index()\n# high_score=test_df[test_df[\"target\"]>0.68]\n# high_score=high_score.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:24.748578Z","iopub.execute_input":"2023-01-01T17:53:24.749064Z","iopub.status.idle":"2023-01-01T17:53:24.753016Z","shell.execute_reply.started":"2023-01-01T17:53:24.749033Z","shell.execute_reply":"2023-01-01T17:53:24.752112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# low_path = '/kaggle/working/low/'\n# high_path = '/kaggle/working/high/'\n# if not os.path.exists(low_path):\n# os.mkdir(low_path)\n# if not os.path.exists(high_path):\n# os.mkdir(high_path)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:25.421174Z","iopub.execute_input":"2023-01-01T17:53:25.42184Z","iopub.status.idle":"2023-01-01T17:53:25.426276Z","shell.execute_reply.started":"2023-01-01T17:53:25.421799Z","shell.execute_reply":"2023-01-01T17:53:25.425174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Low tfrec","metadata":{}},{"cell_type":"code","source":"# if train_only:\n# print('skip the nonstationary model dataset save')\n# else:\n# ct = len(low_score)\n# idx = low_score.index\n# f = 0\n# print(ct)\n# print('Writing TFRecord %i of %i...'%(f,ct))\n# xi=0\n# with tf.io.TFRecordWriter('/kaggle/working/low/test%.2i-%i.tfrec'%(f,ct)) as writer:\n# for _, i in enumerate((tqdm(idx))):\n# r = test_df.iloc[i]\n# file_id = r.id\n# filename=r.cale_fisier\n# img = np.empty((360, width, 2), dtype=np.float32)\n\n# #filename = '%s/test/%s.hdf5' % (di, file_id)\n# with h5py.File(filename, 'r') as f:\n# g = f[file_id]\n# #print('unu')\n# for ch, s in enumerate(['H1', 'L1']):\n# print(xi)\n# xi=xi+1\n# try:\n# a = g[s]['SFTs'][:360, :first_cut] * 1e23 # Fourier coefficient complex64\n# p = 2*(a.real**2 + a.imag**2) # power\n# p /= np.mean(p) # normalize\n# p = np.mean(p.reshape(360, width,compress_1), axis=2)\n# #print('doi')\n# except:\n# a = g[s]['SFTs'][:360, :second_cut]* 1e23 # Fourier coefficient complex64\n# p = 2*(a.real**2 + a.imag**2) # power\n# p /= np.mean(p) # normalize\n# p = np.mean(p.reshape(360, width,compress_2), axis=2)\n# img[...,ch] = p\n\n# # try:\n# # a=g[\"H1\"]\n# # a=a['SFTs'][:360, :4320]*1e22\n# # #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n# # q = np.sin(np.angle(a))*(np.abs(a)**2) # power\n# # q /= np.mean(q) # normalize\n# # q = np.mean(q.reshape(360, 360,12), axis=2)\n# # print(np.min(q),np.max(q))\n# # scaler = MinMaxScaler(feature_range=(0, 255))\n# # scaler = scaler.fit(q)\n# # img[...,2] = q\n# # #cv2.imwrite(\"blabla.jpg\",img)\n# # except:\n# # a=g[\"H1\"]\n# # a=a['SFTs'][:360, :3960]*1e22\n# # p = np.sin(np.angle(a))*(np.abs(a)**2) # power\n# # p /= np.mean(p) # normalize\n# # p = np.mean(p.reshape(360, 360,11), axis=2)\n# # scaler = scaler.fit(p)\n# # img[...,2] = p \n# serialized_img = tf.io.serialize_tensor(img)\n# example = serialize_example_test(serialized_img, str.encode(file_id))\n# writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:25.955092Z","iopub.execute_input":"2023-01-01T17:53:25.955486Z","iopub.status.idle":"2023-01-01T17:53:25.961845Z","shell.execute_reply.started":"2023-01-01T17:53:25.955453Z","shell.execute_reply":"2023-01-01T17:53:25.960537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if train_only:\n# print('skip the nonstationary model dataset save')\n# else:\n# ct = len(high_score)\n# idx = high_score.index\n# f = 0\n# print(ct)\n# print('Writing TFRecord %i of %i...'%(f,ct))\n# xi=0\n# with tf.io.TFRecordWriter('/kaggle/working/high/test%.2i-%i.tfrec'%(f,ct)) as writer:\n# for _, i in enumerate((tqdm(idx))):\n# r = test_df.iloc[i]\n# file_id = r.id\n# filename=r.cale_fisier\n# img = np.empty((360, width, 2), dtype=np.float32)\n\n# #filename = '%s/test/%s.hdf5' % (di, file_id)\n# with h5py.File(filename, 'r') as f:\n# g = f[file_id]\n# #print('unu')\n# for ch, s in enumerate(['H1', 'L1']):\n# print(xi)\n# xi=xi+1\n# try:\n# a = g[s]['SFTs'][:360, :first_cut] * 1e22 # Fourier coefficient complex64\n# p = 2*(a.real**2 + a.imag**2) # power\n# p /= np.mean(p) # normalize\n# p = np.mean(p.reshape(360, width,compress_1), axis=2)\n# #print('doi')\n# except:\n# a = g[s]['SFTs'][:360, :second_cut]* 1e22 # Fourier coefficient complex64\n# p = 2*(a.real**2 + a.imag**2) # power\n# p /= np.mean(p) # normalize\n# p = np.mean(p.reshape(360, width,compress_2), axis=2)\n# img[...,ch] = p\n\n# # try:\n# # a=g[\"H1\"]\n# # a=a['SFTs'][:360, :4320]*1e22\n# # #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n# # q = np.sin(np.angle(a))*(np.abs(a)**2) # power\n# # q /= np.mean(q) # normalize\n# # q = np.mean(q.reshape(360, 360,12), axis=2)\n# # print(np.min(q),np.max(q))\n# # scaler = MinMaxScaler(feature_range=(0, 255))\n# # scaler = scaler.fit(q)\n# # img[...,2] = q\n# # #cv2.imwrite(\"blabla.jpg\",img)\n# # except:\n# # a=g[\"H1\"]\n# # a=a['SFTs'][:360, :3960]*1e22\n# # p = np.sin(np.angle(a))*(np.abs(a)**2) # power\n# # p /= np.mean(p) # normalize\n# # p = np.mean(p.reshape(360, 360,11), axis=2)\n# # scaler = scaler.fit(p)\n# # img[...,2] = p \n# serialized_img = tf.io.serialize_tensor(img)\n# example = serialize_example_test(serialized_img, str.encode(file_id))\n# writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:26.58908Z","iopub.execute_input":"2023-01-01T17:53:26.589471Z","iopub.status.idle":"2023-01-01T17:53:26.597336Z","shell.execute_reply.started":"2023-01-01T17:53:26.58944Z","shell.execute_reply":"2023-01-01T17:53:26.595771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"high tfrec","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}