{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"train_only=True","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.697601Z","iopub.execute_input":"2023-01-03T08:49:25.698016Z","iopub.status.idle":"2023-01-03T08:49:25.729552Z","shell.execute_reply.started":"2023-01-03T08:49:25.697914Z","shell.execute_reply":"2023-01-03T08:49:25.728093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-03T08:49:25.732013Z","iopub.execute_input":"2023-01-03T08:49:25.732783Z","iopub.status.idle":"2023-01-03T08:49:25.74138Z","shell.execute_reply.started":"2023-01-03T08:49:25.73274Z","shell.execute_reply":"2023-01-03T08:49:25.740268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://hal.archives-ouvertes.fr/hal-00085154/document\n\nhttps://hal.archives-ouvertes.fr/hal-00301405/document\n\nhttps://hal.archives-ouvertes.fr/hal-01714325/document","metadata":{}},{"cell_type":"code","source":"#dataset_list=['GS1','GS3','GS2','GS4','GS5','GS6','GS7','GS8','GS9','GS10','GS11','GS12','GS14','GS15']\n#datasetlist\n#GS1-gaussian-noise\n#GS2-nonstationary\n#GS3 narrowartifacts noise\n#GS4-nonstationary\n#GS5 gaussian-noise\n#GS6 gaussian-noise\n#GS7 gaussian-noise\n#GS8 -nonstationary\n#GS9 low SNR gaussian-noise\n#GS10 low SNR gaussian-noise\n#GS11 low SNR gaussian-noise\n#GS12 low SNR nonstationary\n#GS13 low SNR artifacts\n#GS14 low SNR gaussian-noise\n#List SNR short\n###Low only \"pure gaussian noise\"\n#dataset_list=['GS1','GS5','GS6','GS7','GS9','GS10','GS11']\n###High only\n# dataset_list=['GS2','GS3','GS4','GS8','GS10','GS12','GS13']\n#dataset_list=['GS1','GS3','GS2','GS4','GS5','GS6','GS7','GS8','GS9','GS10','GS11','GS12','GS13','GS14','GS15','GS16','GS17','GS18','GS19','GS20']\ndataset_list=['GS9','GS10','GS11','GS12','GS13','GS14','GS15','GS16','GS17','GS18','GS21']\n# dataset_list=['GS21']","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.743336Z","iopub.execute_input":"2023-01-03T08:49:25.744332Z","iopub.status.idle":"2023-01-03T08:49:25.758172Z","shell.execute_reply.started":"2023-01-03T08:49:25.744281Z","shell.execute_reply":"2023-01-03T08:49:25.757145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h5py\nfrom glob import glob\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.760649Z","iopub.execute_input":"2023-01-03T08:49:25.761324Z","iopub.status.idle":"2023-01-03T08:49:25.919367Z","shell.execute_reply.started":"2023-01-03T08:49:25.761279Z","shell.execute_reply":"2023-01-03T08:49:25.918215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib as mpl","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.920627Z","iopub.execute_input":"2023-01-03T08:49:25.92094Z","iopub.status.idle":"2023-01-03T08:49:25.926504Z","shell.execute_reply.started":"2023-01-03T08:49:25.920911Z","shell.execute_reply":"2023-01-03T08:49:25.925247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.927827Z","iopub.execute_input":"2023-01-03T08:49:25.928181Z","iopub.status.idle":"2023-01-03T08:49:25.937066Z","shell.execute_reply.started":"2023-01-03T08:49:25.928151Z","shell.execute_reply":"2023-01-03T08:49:25.935906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport shutil\n\n# Providing the folder path\norigin = '/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'\n#origin=Path(origin)\n# target = '../working/train/'\n# temp='../temp/'\n# if not os.path.exists(temp):\n#     os.mkdir(temp)\n# if not os.path.exists(target):\n#     os.mkdir(target)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.940053Z","iopub.execute_input":"2023-01-03T08:49:25.940534Z","iopub.status.idle":"2023-01-03T08:49:25.949562Z","shell.execute_reply.started":"2023-01-03T08:49:25.940468Z","shell.execute_reply":"2023-01-03T08:49:25.948526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor root, folders, filenames in os.walk('/kaggle/input'):\n   print(root, folders)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:25.951181Z","iopub.execute_input":"2023-01-03T08:49:25.951535Z","iopub.status.idle":"2023-01-03T08:49:41.036698Z","shell.execute_reply.started":"2023-01-03T08:49:25.951504Z","shell.execute_reply":"2023-01-03T08:49:41.035355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_WIDTH=128","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.039204Z","iopub.execute_input":"2023-01-03T08:49:41.039586Z","iopub.status.idle":"2023-01-03T08:49:41.044692Z","shell.execute_reply.started":"2023-01-03T08:49:41.03955Z","shell.execute_reply":"2023-01-03T08:49:41.043491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sft_to_img(sft, ts, buckets=IMG_WIDTH):\n    bucket_size = (ts.max() - ts.min()) // buckets\n    idx = np.searchsorted(ts, [ts[0] + bucket_size * i for i in range(buckets)])\n    # 1. ASD\n    sft = np.absolute(sft)    \n    # 2. shrink\n    global_noise_amp = np.mean(sft, axis=1)\n    img = np.stack([\n        np.mean(i, axis=1) if i.shape[1] > 0 else global_noise_amp for i in np.array_split(sft, idx[1:], axis=1) \n    ])    \n    ts_noise = img.mean(axis=1)\n    # 3. Normalize\n    mean, std = np.mean(img), np.std(img.astype(np.float64))\n    # print(mean, std)    \n\n    img = img - mean\n    img = img / std / 4\n    img *= 128 \n    img += 128\n    # img = img * (255 / np.max(img))\n#     img = np.clip(img, 0, 255).astype(np.uint8)\n    # print(np.min(img), np.max(img), np.mean(img), np.std(img))\n#     return img.T, ts_noise\n    return img.T\n    if DEBUG:\n        img, ts_noise = sft_to_img(h1_sft, h1_ts)\n        plt.figure()\n        plt.imshow(img, cmap='gray')\n        plt.figure()\n        plt.plot(ts_noise)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.045984Z","iopub.execute_input":"2023-01-03T08:49:41.046314Z","iopub.status.idle":"2023-01-03T08:49:41.061291Z","shell.execute_reply.started":"2023-01-03T08:49:41.046283Z","shell.execute_reply":"2023-01-03T08:49:41.060305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_2d(matrix):\n    # Only this is changed to use 2-norm put 2 instead of 1\n    norm = np.linalg.norm(matrix, 1)\n    # normalized matrix\n    matrix = matrix/norm\n    \n    return matrix\n# Utility to read hdf5 file\ndef read_data(file: Path):\n    with h5py.File(file, \"r\") as f:\n        file=Path(file)\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        #freq_hz = list(f[\"frequency_Hz\"])\n        ###\n        h1_stft = h1[\"SFTs\"][()]\n        ###\n        #h1_timestamp = h1[\"timestamps_GPS\"][()]\n        h1_timestamp =1\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        #l1_timestamp = l1[\"timestamps_GPS\"][()]\n        l1_timestamp =2\n        return {\n            \"H1\": [h1_stft, h1_timestamp],\n            \"L1\": [l1_stft, l1_timestamp],\n            #\"freq_hz\": freq_hz\n        }\ndef read_data_c(file):\n    file = Path(file)\n    with h5py.File(file, \"r\") as f:\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        freq_hz = f[\"frequency_Hz\"]\n        h1_stft = np.array(f['H1']['SFTs'], dtype=np.complex128)\n#         h1_stft = h1[\"SFTs\"][()]\n        h1_timestamp = h1[\"timestamps_GPS\"][()]\n        # H2 data\n#         l1_stft = l1[\"SFTs\"][()]\n        l1_stft = np.array(f['L1']['SFTs'], dtype=np.complex128)\n        l1_timestamp = l1[\"timestamps_GPS\"][()]\n        \n        return [h1_stft, h1_timestamp],            [l1_stft, l1_timestamp], np.array(freq_hz)\n\ndef power_spectrogram2(h1_sft,l1_sft):\n    i=0\n    \n    img = np.empty((360,360,3), dtype=np.float64)\n    try:\n        \n        a=h1_sft[:360, :4320]\n        p = a.real**2 + a.imag**2\n        p = normalize_2d(p)\n        p /= np.mean(p)  # normalize\n        q=(a.real*2)*(np.angle(a))\n        q=normalize_2d(q)\n#         q /= np.mean(q)  # normalize\n        try:\n            p = np.mean(p.reshape(360, 360,12), axis=2)\n            q = np.mean(q.reshape(360, 360,12), axis=2)\n        except:\n            a=h1_sft[:360, :3960]\n            p = a.real**2 + a.imag**2\n            p=normalize_2d(p)\n            q=(a.real*2)*(np.angle(a))\n            q=normalize_2d(q)\n#             p /= np.mean(p)  # normalize\n#             q /= np.mean(q)  # normalize\n            p = np.mean(p.reshape(360, 360,11), axis=2)\n            q = np.mean(q.reshape(360, 360,11), axis=2)\n\n        #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n\n    #     normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n        img[:,:,0]= p*255\n#         img[:,:,2]= np.ones((360,360), dtype=np.float32)\n#         img[:,:,2]=img[:,:,2]*255\n\n\n        img[:,:,2]= q*255\n        a=l1_sft[:360, :4320]\n        p = a.real**2 + a.imag**2\n        p=normalize_2d(p)\n#         p /= np.mean(p)  # normalize\n        try:\n            p = np.mean(p.reshape(360, 360,12), axis=2)\n        except:\n            a=l1_sft[:360, :3960]\n            p = a.real**2 + a.imag**2\n            p=normalize_2d(p)\n#             p /= np.mean(p)  # normalize\n            p = np.mean(p.reshape(360, 360,11), axis=2)\n\n        #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n        img[:,:,1]= p*255\n\n#         img = np.moveaxis(img, 0, -1)\n        #return np.asarray([img[0],img[1]]).astype('float32')\n    except:\n        print(\"empty image\")\n#         img = np.moveaxis(img, 0, -1)\n\n    return img   \ndef plot_spectrogram(h1_sft,l1_sft,width):\n    img = np.empty((360,width,3), dtype=np.float64)\n    a=h1_sft\n    p = a.real**2 + a.imag**2\n    p=normalize_2d(p)\n    q=(a.real*2)*(np.angle(a))\n    q=normalize_2d(q)\n    img[:,:,0]= p*255\n    img[:,:,1]= p*255\n    try:\n        a=l1_sft\n        p = a.real**2 + a.imag**2\n        p=normalize_2d(p)\n        img[:,:,2]= p*255\n    except:\n        print('bad easter egg')\n    return img\n        \n#     plt.imshow(img)\n#     plt.show()\n    \ndef power_spectrogram3(h1_sft,l1_sft):\n    i=0\n    \n    img = np.empty((360,360,3), dtype=np.float32)\n    try:\n        a=h1_sft[:360, :4320]*1e22\n        p = a.real**2 + a.imag**2\n        p /= np.mean(p)  # normalize\n        q=(a.real*2)*(np.angle(a))\n        q /= np.mean(q)  # normalize\n        try:\n            p = np.mean(p.reshape(360, 360,12), axis=2)\n            q = np.mean(q.reshape(360, 360,12), axis=2)\n        except:\n            a=h1_sft[:360, :3960]*1e22\n            p = a.real**2 + a.imag**2\n            q=(a.real*2)*(np.angle(a))\n            p /= np.mean(p)  # normalize\n            q /= np.mean(q)  # normalize\n            p = np.mean(p.reshape(360, 360,11), axis=2)\n            q = np.mean(q.reshape(360, 360,11), axis=2)\n\n        #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n\n    #     normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n        img[:,:,0]= p*255\n#         img[:,:,2]= np.ones((360,360), dtype=np.float32)\n#         img[:,:,2]=img[:,:,2]*255\n\n\n        img[:,:,2]= q*255\n        a=l1_sft[:360, :4320]*1e22\n        p = a.real**2 + a.imag**2\n        p /= np.mean(p)  # normalize\n        try:\n            p = np.mean(p.reshape(360, 360,12), axis=2)\n        except:\n            a=l1_sft[:360, :3960]*1e22\n            p = a.real**2 + a.imag**2\n            p /= np.mean(p)  # normalize\n            p = np.mean(p.reshape(360, 360,11), axis=2)\n\n        #normalized=(255*(p - np.min(p))/np.ptp(p)).astype(int)\n        img[:,:,1]= p*255\n\n#         img = np.moveaxis(img, 0, -1)\n        #return np.asarray([img[0],img[1]]).astype('float32')\n    except:\n        print(\"empty image\")\n#         img = np.moveaxis(img, 0, -1)\n\n    return img\ndef show_power_spectrogram2(filename):\n    f = h5py.File(filename, 'r')\n\n  # read fourier transform coefficients\n    (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n    width=len(h1_sfts)\n    img=power_spectrogram2(h1_sfts,l1_sfts,)\n    plt.imshow(img)\n    plt.show()\ndef show_power_spectrogram(filename):\n    f = h5py.File(filename, 'r')\n\n  # read fourier transform coefficients\n    (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n    width=h1_sfts.shape[1]\n    img=plot_spectrogram(h1_sfts,l1_sfts,width)\n    plt.rcParams[\"figure.figsize\"] = (15,5)\n    plt.imshow(img)\n    plt.show()\ndef test_dataset(filename):\n    filename=Path(filename)\n    with h5py.File(filename) as f:\n                filename=filename.stem\n                g = f[filename]\n                print(filename)\n                try:\n                    for ch, s in enumerate(['H1', 'L1']):\n                            x=g[s]['SFTs'].shape\n                except:\n                    print('bad dataset')","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.062836Z","iopub.execute_input":"2023-01-03T08:49:41.063411Z","iopub.status.idle":"2023-01-03T08:49:41.101502Z","shell.execute_reply.started":"2023-01-03T08:49:41.063378Z","shell.execute_reply":"2023-01-03T08:49:41.100278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1+1","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.103007Z","iopub.execute_input":"2023-01-03T08:49:41.103372Z","iopub.status.idle":"2023-01-03T08:49:41.123222Z","shell.execute_reply.started":"2023-01-03T08:49:41.10334Z","shell.execute_reply":"2023-01-03T08:49:41.122252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Fetching the list of all the files\n# files = os.listdir(origin)\n\n# # Fetching all the files to directory\n# for file_name in files:\n#     shutil.copy(origin+file_name, target+file_name)\n# print(\"Files are copied successfully\")|","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.124982Z","iopub.execute_input":"2023-01-03T08:49:41.125341Z","iopub.status.idle":"2023-01-03T08:49:41.132637Z","shell.execute_reply.started":"2023-01-03T08:49:41.125309Z","shell.execute_reply":"2023-01-03T08:49:41.131495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"heigh=360\n# width=360\n# first_cut=4320 #360\n# compress_1=12\n# second_cut=3960 #360\n# compress_2=11\nwidth=128\nfirst_cut=4096\ncompress_1=32\nsecond_cut=4096-128\ncompress_2=31","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.13707Z","iopub.execute_input":"2023-01-03T08:49:41.137409Z","iopub.status.idle":"2023-01-03T08:49:41.147916Z","shell.execute_reply.started":"2023-01-03T08:49:41.137379Z","shell.execute_reply":"2023-01-03T08:49:41.146927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/train_labels.csv\")\nlen(train)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.149549Z","iopub.execute_input":"2023-01-03T08:49:41.149921Z","iopub.status.idle":"2023-01-03T08:49:41.171046Z","shell.execute_reply.started":"2023-01-03T08:49:41.149885Z","shell.execute_reply":"2023-01-03T08:49:41.170058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.172306Z","iopub.execute_input":"2023-01-03T08:49:41.172664Z","iopub.status.idle":"2023-01-03T08:49:41.18778Z","shell.execute_reply.started":"2023-01-03T08:49:41.172632Z","shell.execute_reply":"2023-01-03T08:49:41.186711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['cale_fisier']='/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'+train['id']+'.hdf5'","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.189356Z","iopub.execute_input":"2023-01-03T08:49:41.190198Z","iopub.status.idle":"2023-01-03T08:49:41.198876Z","shell.execute_reply.started":"2023-01-03T08:49:41.190129Z","shell.execute_reply":"2023-01-03T08:49:41.198075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train['cale_fisier']=train['target'].apply(x:'/kaggle/input/g2net-detecting-continuous-gravitational-waves/'+train['id']+'.hdf5' if x=1.0 else '/kaggle/input/g2net-detecting-continuous-gravitational-waves/'+train['id']+'.hdf5')\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.199746Z","iopub.execute_input":"2023-01-03T08:49:41.200051Z","iopub.status.idle":"2023-01-03T08:49:41.210511Z","shell.execute_reply.started":"2023-01-03T08:49:41.200022Z","shell.execute_reply":"2023-01-03T08:49:41.209492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram2(train.cale_fisier[1])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:41.213841Z","iopub.execute_input":"2023-01-03T08:49:41.214157Z","iopub.status.idle":"2023-01-03T08:49:42.601191Z","shell.execute_reply.started":"2023-01-03T08:49:41.214128Z","shell.execute_reply":"2023-01-03T08:49:42.600082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Easter eggs","metadata":{}},{"cell_type":"code","source":"easter_eggs=train[train.target<0]","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:42.602508Z","iopub.execute_input":"2023-01-03T08:49:42.603041Z","iopub.status.idle":"2023-01-03T08:49:42.609725Z","shell.execute_reply.started":"2023-01-03T08:49:42.603008Z","shell.execute_reply":"2023-01-03T08:49:42.608649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"easter_eggs=easter_eggs.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:42.611173Z","iopub.execute_input":"2023-01-03T08:49:42.61161Z","iopub.status.idle":"2023-01-03T08:49:42.620163Z","shell.execute_reply.started":"2023-01-03T08:49:42.611577Z","shell.execute_reply":"2023-01-03T08:49:42.619288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"easter_eggs.cale_fisier","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:42.621692Z","iopub.execute_input":"2023-01-03T08:49:42.622186Z","iopub.status.idle":"2023-01-03T08:49:42.634083Z","shell.execute_reply.started":"2023-01-03T08:49:42.622154Z","shell.execute_reply":"2023-01-03T08:49:42.632843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(train.cale_fisier[3])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:42.635582Z","iopub.execute_input":"2023-01-03T08:49:42.636114Z","iopub.status.idle":"2023-01-03T08:49:44.419814Z","shell.execute_reply.started":"2023-01-03T08:49:42.636081Z","shell.execute_reply":"2023-01-03T08:49:44.418506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(easter_eggs.cale_fisier[0])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:44.421085Z","iopub.execute_input":"2023-01-03T08:49:44.42204Z","iopub.status.idle":"2023-01-03T08:49:44.940574Z","shell.execute_reply.started":"2023-01-03T08:49:44.421989Z","shell.execute_reply":"2023-01-03T08:49:44.939508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(easter_eggs.cale_fisier[2])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:44.941835Z","iopub.execute_input":"2023-01-03T08:49:44.942152Z","iopub.status.idle":"2023-01-03T08:49:45.446937Z","shell.execute_reply.started":"2023-01-03T08:49:44.942123Z","shell.execute_reply":"2023-01-03T08:49:45.445797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram(easter_eggs.cale_fisier[1])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:45.448256Z","iopub.execute_input":"2023-01-03T08:49:45.448954Z","iopub.status.idle":"2023-01-03T08:49:46.140854Z","shell.execute_reply.started":"2023-01-03T08:49:45.448916Z","shell.execute_reply":"2023-01-03T08:49:46.139674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_power_spectrogram2(easter_eggs.cale_fisier[2])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.142175Z","iopub.execute_input":"2023-01-03T08:49:46.142537Z","iopub.status.idle":"2023-01-03T08:49:46.482764Z","shell.execute_reply.started":"2023-01-03T08:49:46.142506Z","shell.execute_reply":"2023-01-03T08:49:46.481665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.cale_fisier[555]","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.484008Z","iopub.execute_input":"2023-01-03T08:49:46.484341Z","iopub.status.idle":"2023-01-03T08:49:46.490754Z","shell.execute_reply.started":"2023-01-03T08:49:46.48431Z","shell.execute_reply":"2023-01-03T08:49:46.489682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.498955Z","iopub.execute_input":"2023-01-03T08:49:46.499338Z","iopub.status.idle":"2023-01-03T08:49:46.507195Z","shell.execute_reply.started":"2023-01-03T08:49:46.499303Z","shell.execute_reply":"2023-01-03T08:49:46.506002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2nd-addon-g2netgaussian-noise-dataset\n\n","metadata":{}},{"cell_type":"code","source":"dataset_number='GS6'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/2nd-addon-g2netgaussian-noise-dataset/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.509318Z","iopub.execute_input":"2023-01-03T08:49:46.510283Z","iopub.status.idle":"2023-01-03T08:49:46.51864Z","shell.execute_reply.started":"2023-01-03T08:49:46.510234Z","shell.execute_reply":"2023-01-03T08:49:46.517503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS6'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/2nd-addon-g2netgaussian-noise-dataset/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.520258Z","iopub.execute_input":"2023-01-03T08:49:46.520602Z","iopub.status.idle":"2023-01-03T08:49:46.529701Z","shell.execute_reply.started":"2023-01-03T08:49:46.520571Z","shell.execute_reply":"2023-01-03T08:49:46.528676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3rdaddon-g2netgaussian-noise-dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS7'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/3rdaddon-g2netgaussian-noise-dataset/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.531279Z","iopub.execute_input":"2023-01-03T08:49:46.532525Z","iopub.status.idle":"2023-01-03T08:49:46.546453Z","shell.execute_reply.started":"2023-01-03T08:49:46.532477Z","shell.execute_reply":"2023-01-03T08:49:46.545158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS7'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/3rdaddon-g2netgaussian-noise-dataset/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.548188Z","iopub.execute_input":"2023-01-03T08:49:46.548645Z","iopub.status.idle":"2023-01-03T08:49:46.559184Z","shell.execute_reply.started":"2023-01-03T08:49:46.548602Z","shell.execute_reply":"2023-01-03T08:49:46.558182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ND-ADDon G2Net-Non-stationary Dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS8'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/ndaddon-g2netnonstationary-dataset/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.560331Z","iopub.execute_input":"2023-01-03T08:49:46.560774Z","iopub.status.idle":"2023-01-03T08:49:46.57154Z","shell.execute_reply.started":"2023-01-03T08:49:46.560732Z","shell.execute_reply":"2023-01-03T08:49:46.570739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS8'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/ndaddon-g2netnonstationary-dataset/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])\n    ","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.572752Z","iopub.execute_input":"2023-01-03T08:49:46.573047Z","iopub.status.idle":"2023-01-03T08:49:46.583734Z","shell.execute_reply.started":"2023-01-03T08:49:46.573019Z","shell.execute_reply":"2023-01-03T08:49:46.582687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LowSNRG2Net-Gaussian Noise Dataset","metadata":{}},{"cell_type":"code","source":"dataset_number='GS9'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/lowsnrg2net-gaussian-noise-dataset/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:46.585313Z","iopub.execute_input":"2023-01-03T08:49:46.585646Z","iopub.status.idle":"2023-01-03T08:49:47.857843Z","shell.execute_reply.started":"2023-01-03T08:49:46.585617Z","shell.execute_reply":"2023-01-03T08:49:47.856929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS9'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/lowsnrg2net-gaussian-noise-dataset/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:47.859094Z","iopub.execute_input":"2023-01-03T08:49:47.859642Z","iopub.status.idle":"2023-01-03T08:49:49.16631Z","shell.execute_reply.started":"2023-01-03T08:49:47.859606Z","shell.execute_reply":"2023-01-03T08:49:49.165106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### GeneratinglowSNRgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS10'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/generatinglowsnrgravitywaves/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:49.168048Z","iopub.execute_input":"2023-01-03T08:49:49.168403Z","iopub.status.idle":"2023-01-03T08:49:49.180429Z","shell.execute_reply.started":"2023-01-03T08:49:49.16837Z","shell.execute_reply":"2023-01-03T08:49:49.179292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS10'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/generatinglowsnrgravitywaves/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:49.182111Z","iopub.execute_input":"2023-01-03T08:49:49.18245Z","iopub.status.idle":"2023-01-03T08:49:50.449419Z","shell.execute_reply.started":"2023-01-03T08:49:49.182404Z","shell.execute_reply":"2023-01-03T08:49:50.448177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2ndgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS11'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/2ndgeneratinglowsnrgravitywaves/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:50.451079Z","iopub.execute_input":"2023-01-03T08:49:50.451605Z","iopub.status.idle":"2023-01-03T08:49:50.467179Z","shell.execute_reply.started":"2023-01-03T08:49:50.451563Z","shell.execute_reply":"2023-01-03T08:49:50.465767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS11'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/2ndgeneratinglowsnrgravitywaves/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:50.468571Z","iopub.execute_input":"2023-01-03T08:49:50.468962Z","iopub.status.idle":"2023-01-03T08:49:51.708065Z","shell.execute_reply.started":"2023-01-03T08:49:50.468929Z","shell.execute_reply":"2023-01-03T08:49:51.70694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### generatinglowsnrnonstationarygravitywaves\n\n","metadata":{}},{"cell_type":"code","source":"dataset_number='GS12'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/generatinglowsnrnonstationarygravitywaves/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:51.709791Z","iopub.execute_input":"2023-01-03T08:49:51.710561Z","iopub.status.idle":"2023-01-03T08:49:51.721919Z","shell.execute_reply.started":"2023-01-03T08:49:51.710525Z","shell.execute_reply":"2023-01-03T08:49:51.720865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS12'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/generatinglowsnrnonstationarygravitywaves/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:51.723335Z","iopub.execute_input":"2023-01-03T08:49:51.723691Z","iopub.status.idle":"2023-01-03T08:49:53.018025Z","shell.execute_reply.started":"2023-01-03T08:49:51.723659Z","shell.execute_reply":"2023-01-03T08:49:53.016487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### set3-generating-low-snr-nonstationary","metadata":{}},{"cell_type":"code","source":"dataset_number='GS13'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/set3-generating-low-snr-nonstationary/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:53.019556Z","iopub.execute_input":"2023-01-03T08:49:53.020588Z","iopub.status.idle":"2023-01-03T08:49:53.035138Z","shell.execute_reply.started":"2023-01-03T08:49:53.020543Z","shell.execute_reply":"2023-01-03T08:49:53.034241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS13'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/set3-generating-low-snr-nonstationary/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:53.036363Z","iopub.execute_input":"2023-01-03T08:49:53.037459Z","iopub.status.idle":"2023-01-03T08:49:54.417712Z","shell.execute_reply.started":"2023-01-03T08:49:53.037409Z","shell.execute_reply":"2023-01-03T08:49:54.416532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### snrlowgeavitywavessimulation","metadata":{}},{"cell_type":"code","source":"dataset_number='GS14'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/snrlowgeavitywavessimulation/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:54.419218Z","iopub.execute_input":"2023-01-03T08:49:54.420385Z","iopub.status.idle":"2023-01-03T08:49:54.433882Z","shell.execute_reply.started":"2023-01-03T08:49:54.420342Z","shell.execute_reply":"2023-01-03T08:49:54.43246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS14'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/snrlowgeavitywavessimulation/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:54.43561Z","iopub.execute_input":"2023-01-03T08:49:54.436105Z","iopub.status.idle":"2023-01-03T08:49:55.602488Z","shell.execute_reply.started":"2023-01-03T08:49:54.436059Z","shell.execute_reply":"2023-01-03T08:49:55.601293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## lowsnrnonstationary","metadata":{}},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/lowsnrnonstationary/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:55.603978Z","iopub.execute_input":"2023-01-03T08:49:55.604468Z","iopub.status.idle":"2023-01-03T08:49:55.620635Z","shell.execute_reply.started":"2023-01-03T08:49:55.604409Z","shell.execute_reply":"2023-01-03T08:49:55.619482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/lowsnrnonstationary/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:55.622094Z","iopub.execute_input":"2023-01-03T08:49:55.622447Z","iopub.status.idle":"2023-01-03T08:49:56.916714Z","shell.execute_reply.started":"2023-01-03T08:49:55.622399Z","shell.execute_reply":"2023-01-03T08:49:56.915467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### artifactsgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/artifactsgeneratinglowsnrgravitywaves/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:56.9181Z","iopub.execute_input":"2023-01-03T08:49:56.918767Z","iopub.status.idle":"2023-01-03T08:49:56.931795Z","shell.execute_reply.started":"2023-01-03T08:49:56.918726Z","shell.execute_reply":"2023-01-03T08:49:56.930451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS15'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/artifactsgeneratinglowsnrgravitywaves/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:56.93323Z","iopub.execute_input":"2023-01-03T08:49:56.933788Z","iopub.status.idle":"2023-01-03T08:49:58.226957Z","shell.execute_reply.started":"2023-01-03T08:49:56.933742Z","shell.execute_reply":"2023-01-03T08:49:58.225522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2ndartifactsgeneratinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS16'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/2ndartifactsgeneratinglowsnrgravitywaves/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:58.228464Z","iopub.execute_input":"2023-01-03T08:49:58.229051Z","iopub.status.idle":"2023-01-03T08:49:58.241159Z","shell.execute_reply.started":"2023-01-03T08:49:58.229016Z","shell.execute_reply":"2023-01-03T08:49:58.240381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS16'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/2ndartifactsgeneratinglowsnrgravitywaves/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:58.242349Z","iopub.execute_input":"2023-01-03T08:49:58.243257Z","iopub.status.idle":"2023-01-03T08:49:59.494619Z","shell.execute_reply.started":"2023-01-03T08:49:58.243221Z","shell.execute_reply":"2023-01-03T08:49:59.493753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### trimmedg2dataset1","metadata":{}},{"cell_type":"code","source":"dataset_number='GS17'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/trimmedg2dataset1/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:59.496062Z","iopub.execute_input":"2023-01-03T08:49:59.496704Z","iopub.status.idle":"2023-01-03T08:49:59.509152Z","shell.execute_reply.started":"2023-01-03T08:49:59.496669Z","shell.execute_reply":"2023-01-03T08:49:59.508073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS17'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/trimmedg2dataset1/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:49:59.510953Z","iopub.execute_input":"2023-01-03T08:49:59.511672Z","iopub.status.idle":"2023-01-03T08:50:00.7761Z","shell.execute_reply.started":"2023-01-03T08:49:59.511628Z","shell.execute_reply":"2023-01-03T08:50:00.774948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3ndartifactsgeneratinglowsnrgravit","metadata":{}},{"cell_type":"code","source":"dataset_number='GS18'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/3ndartifactsgeneratinglowsnrgravit/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:00.777501Z","iopub.execute_input":"2023-01-03T08:50:00.777849Z","iopub.status.idle":"2023-01-03T08:50:00.791429Z","shell.execute_reply.started":"2023-01-03T08:50:00.777818Z","shell.execute_reply":"2023-01-03T08:50:00.790308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS18'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/3ndartifactsgeneratinglowsnrgravit/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:00.79321Z","iopub.execute_input":"2023-01-03T08:50:00.793703Z","iopub.status.idle":"2023-01-03T08:50:02.021621Z","shell.execute_reply.started":"2023-01-03T08:50:00.793659Z","shell.execute_reply":"2023-01-03T08:50:02.020498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### trimmed2generatinglowsnrgravitywaves","metadata":{}},{"cell_type":"code","source":"dataset_number='GS19'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/trimmed2generatinglowsnrgravitywaves/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.022781Z","iopub.execute_input":"2023-01-03T08:50:02.023097Z","iopub.status.idle":"2023-01-03T08:50:02.03005Z","shell.execute_reply.started":"2023-01-03T08:50:02.023068Z","shell.execute_reply":"2023-01-03T08:50:02.028928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_number='GS19'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/trimmed2generatinglowsnrgravitywaves/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.03157Z","iopub.execute_input":"2023-01-03T08:50:02.031987Z","iopub.status.idle":"2023-01-03T08:50:02.042549Z","shell.execute_reply.started":"2023-01-03T08:50:02.031944Z","shell.execute_reply":"2023-01-03T08:50:02.041494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### set4generatinglowsnrnonstationary","metadata":{}},{"cell_type":"code","source":"dataset_number='GS20'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/set4generatinglowsnrnonstationary/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.044162Z","iopub.execute_input":"2023-01-03T08:50:02.045043Z","iopub.status.idle":"2023-01-03T08:50:02.060164Z","shell.execute_reply.started":"2023-01-03T08:50:02.045006Z","shell.execute_reply":"2023-01-03T08:50:02.058918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"dataset_number='GS20'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/set4generatinglowsnrnonstationary/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['cale_fisier']=add_on+df_add.id\n    train=pd.concat([train, df_add])\n    train=train.reset_index(drop=True)\n    show_power_spectrogram2(df_add.cale_fisier[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.064123Z","iopub.execute_input":"2023-01-03T08:50:02.064761Z","iopub.status.idle":"2023-01-03T08:50:02.072696Z","shell.execute_reply.started":"2023-01-03T08:50:02.064724Z","shell.execute_reply":"2023-01-03T08:50:02.071463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')\ntrain=train[train['target'] >=0]\ntrain=train.reset_index()\ntrain = train.sample(frac=1)\ntrain=train.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.074331Z","iopub.execute_input":"2023-01-03T08:50:02.075257Z","iopub.status.idle":"2023-01-03T08:50:02.096818Z","shell.execute_reply.started":"2023-01-03T08:50:02.075211Z","shell.execute_reply":"2023-01-03T08:50:02.095378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.098176Z","iopub.execute_input":"2023-01-03T08:50:02.098534Z","iopub.status.idle":"2023-01-03T08:50:02.110842Z","shell.execute_reply.started":"2023-01-03T08:50:02.0985Z","shell.execute_reply":"2023-01-03T08:50:02.109781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['filetype']=0","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.11212Z","iopub.execute_input":"2023-01-03T08:50:02.11243Z","iopub.status.idle":"2023-01-03T08:50:02.120141Z","shell.execute_reply.started":"2023-01-03T08:50:02.112402Z","shell.execute_reply":"2023-01-03T08:50:02.119096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.cale_fisier[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.121351Z","iopub.execute_input":"2023-01-03T08:50:02.1218Z","iopub.status.idle":"2023-01-03T08:50:02.132022Z","shell.execute_reply.started":"2023-01-03T08:50:02.121768Z","shell.execute_reply":"2023-01-03T08:50:02.131254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.132903Z","iopub.execute_input":"2023-01-03T08:50:02.133192Z","iopub.status.idle":"2023-01-03T08:50:02.14431Z","shell.execute_reply.started":"2023-01-03T08:50:02.133165Z","shell.execute_reply":"2023-01-03T08:50:02.143209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndataset_number='GS21'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/sinteticsignals/noise/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=0\n    df_add['filetype']=1\n    df_add['cale_fisier']=add_on+df_add.id","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.145793Z","iopub.execute_input":"2023-01-03T08:50:02.146124Z","iopub.status.idle":"2023-01-03T08:50:02.164021Z","shell.execute_reply.started":"2023-01-03T08:50:02.146095Z","shell.execute_reply":"2023-01-03T08:50:02.162951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.165502Z","iopub.execute_input":"2023-01-03T08:50:02.166262Z","iopub.status.idle":"2023-01-03T08:50:02.178287Z","shell.execute_reply.started":"2023-01-03T08:50:02.166227Z","shell.execute_reply":"2023-01-03T08:50:02.17727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.concat([train, df_add])\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.179471Z","iopub.execute_input":"2023-01-03T08:50:02.180379Z","iopub.status.idle":"2023-01-03T08:50:02.195727Z","shell.execute_reply.started":"2023-01-03T08:50:02.180333Z","shell.execute_reply":"2023-01-03T08:50:02.194728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndataset_number='GS21'\nif dataset_number in dataset_list:\n    print('ok:',dataset_number)\n    add_on = \"/kaggle/input/sinteticsignals/signal/\"\n    files_add_on = os.listdir(add_on)\n    df_add=pd.DataFrame()   \n    df_add=pd.DataFrame(files_add_on)\n    df_add=df_add.rename(columns={0: \"id\"})\n    df_add['target']=1\n    df_add['filetype']=1\n    df_add['cale_fisier']=add_on+df_add.id","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.197389Z","iopub.execute_input":"2023-01-03T08:50:02.1983Z","iopub.status.idle":"2023-01-03T08:50:02.21456Z","shell.execute_reply.started":"2023-01-03T08:50:02.198256Z","shell.execute_reply":"2023-01-03T08:50:02.213543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_add.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.21572Z","iopub.execute_input":"2023-01-03T08:50:02.21677Z","iopub.status.idle":"2023-01-03T08:50:02.227139Z","shell.execute_reply.started":"2023-01-03T08:50:02.216735Z","shell.execute_reply":"2023-01-03T08:50:02.226196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.concat([train, df_add])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.228386Z","iopub.execute_input":"2023-01-03T08:50:02.229327Z","iopub.status.idle":"2023-01-03T08:50:02.238401Z","shell.execute_reply.started":"2023-01-03T08:50:02.229288Z","shell.execute_reply":"2023-01-03T08:50:02.237137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train= train.sample(frac=1)\ntrain=train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.239903Z","iopub.execute_input":"2023-01-03T08:50:02.240464Z","iopub.status.idle":"2023-01-03T08:50:02.25308Z","shell.execute_reply.started":"2023-01-03T08:50:02.240411Z","shell.execute_reply":"2023-01-03T08:50:02.252015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# short_train_testing=train.sample(100)\n# short_train_testing=short_train_testing.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.262201Z","iopub.execute_input":"2023-01-03T08:50:02.262642Z","iopub.status.idle":"2023-01-03T08:50:02.267734Z","shell.execute_reply.started":"2023-01-03T08:50:02.262603Z","shell.execute_reply":"2023-01-03T08:50:02.266618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T08:50:02.268973Z","iopub.execute_input":"2023-01-03T08:50:02.269558Z","iopub.status.idle":"2023-01-03T08:50:02.281255Z","shell.execute_reply.started":"2023-01-03T08:50:02.269525Z","shell.execute_reply":"2023-01-03T08:50:02.280136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 5\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:45:47.194131Z","iopub.execute_input":"2023-01-02T18:45:47.194511Z","iopub.status.idle":"2023-01-02T18:45:47.200081Z","shell.execute_reply.started":"2023-01-02T18:45:47.19448Z","shell.execute_reply":"2023-01-02T18:45:47.198974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:45:50.321135Z","iopub.execute_input":"2023-01-02T18:45:50.321859Z","iopub.status.idle":"2023-01-02T18:45:55.084559Z","shell.execute_reply.started":"2023-01-02T18:45:50.321823Z","shell.execute_reply":"2023-01-02T18:45:55.083423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:07.986994Z","iopub.execute_input":"2023-01-02T18:46:07.987419Z","iopub.status.idle":"2023-01-02T18:46:08.590834Z","shell.execute_reply.started":"2023-01-02T18:46:07.987384Z","shell.execute_reply":"2023-01-02T18:46:08.589933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = StratifiedKFold(n_splits=FOLDS, random_state=SEED, shuffle=True)\nfor i, (train_idx, valid_idx) in enumerate(kfold.split(train, train.target)): \n    train.loc[valid_idx, 'fold'] = int(i)\nfolds = train \ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:10.934944Z","iopub.execute_input":"2023-01-02T18:46:10.93535Z","iopub.status.idle":"2023-01-02T18:46:10.962493Z","shell.execute_reply.started":"2023-01-02T18:46:10.93532Z","shell.execute_reply":"2023-01-02T18:46:10.961469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:14.743766Z","iopub.execute_input":"2023-01-02T18:46:14.744927Z","iopub.status.idle":"2023-01-02T18:46:14.753476Z","shell.execute_reply.started":"2023-01-02T18:46:14.744879Z","shell.execute_reply":"2023-01-02T18:46:14.75218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"serialize test and train","metadata":{}},{"cell_type":"code","source":"def serialize_example_train(img, tgt, name):\n  feature = {\n      'spectrogram': _bytes_feature(img),\n      'target': _float_feature(tgt),\n      'id': _bytes_feature(name),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()\n\ndef serialize_example_test(img, name):\n  feature = {\n      'spectrogram': _bytes_feature(img),\n      'id': _bytes_feature(name),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:17.026245Z","iopub.execute_input":"2023-01-02T18:46:17.026715Z","iopub.status.idle":"2023-01-02T18:46:17.034099Z","shell.execute_reply.started":"2023-01-02T18:46:17.026664Z","shell.execute_reply":"2023-01-02T18:46:17.032953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler(feature_range=(0, 255))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:46:17.929481Z","iopub.execute_input":"2023-01-02T18:46:17.930206Z","iopub.status.idle":"2023-01-02T18:46:17.935849Z","shell.execute_reply.started":"2023-01-02T18:46:17.930157Z","shell.execute_reply":"2023-01-02T18:46:17.934693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import cv2","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-01T17:55:29.125993Z","iopub.execute_input":"2023-01-01T17:55:29.12638Z","iopub.status.idle":"2023-01-01T17:55:29.131252Z","shell.execute_reply.started":"2023-01-01T17:55:29.126351Z","shell.execute_reply":"2023-01-01T17:55:29.130112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img = cv2.imread('blabla.jpg')\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T17:55:29.62355Z","iopub.execute_input":"2023-01-01T17:55:29.624541Z","iopub.status.idle":"2023-01-01T17:55:29.628015Z","shell.execute_reply.started":"2023-01-01T17:55:29.624506Z","shell.execute_reply":"2023-01-01T17:55:29.627289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# b,g,r = cv2.split(img)\n# k = np.zeros_like(b)\n# # b = cv2.merge([b,k,k])\n# # g = cv2.merge([k,g,k])\n# # r = cv2.merge([k,k,r])\n# #Show the image with matplotlib\n# plt.imshow(b)\n# plt.show()\n# plt.imshow(g)\n# plt.show()\n# plt.imshow(r)\n# plt.show()\n# # plt.imshow(\"red\",r)\n# # cv2.imshow(\"green\",g)\n# # cv2.imshow(\"blue\",b)\n# # cv2.waitKey(0)\n# # cv2.destroyAllWindows()","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-01T17:55:30.223615Z","iopub.execute_input":"2023-01-01T17:55:30.224025Z","iopub.status.idle":"2023-01-01T17:55:30.229539Z","shell.execute_reply.started":"2023-01-01T17:55:30.223994Z","shell.execute_reply":"2023-01-01T17:55:30.228329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in range(FOLDS):\n    ct = (folds['fold'] == f).sum()\n    idx = folds[folds['fold'] == f].index\n\n    print(ct)\n    print('Writing TFRecord %i of %i...'%(f,ct))\n    \n    with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(f,ct)) as writer:\n        for _, i in enumerate((tqdm(idx))):\n            r = folds.iloc[i]\n            y = np.float32(r.target)\n            file_id = r.id\n            filename=r.cale_fisier\n            if r.filetype==0:\n                img = np.empty((360, width, 2), dtype=np.float64)\n                (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n                imagine=sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)\n                img[...,0] = imagine\n                imagine=sft_to_img(l1_sfts, l1_ts, buckets=IMG_WIDTH)\n                img[...,1] = imagine\n            else:\n                img=np.load(filename)\n                img=img.astype('float64')\n    #             filename = '%s/train/%s.hdf5' % (di, file_id)\n    #             filename=\n    #             try:\n    #                 with h5py.File(filename, 'r') as f:\n    #                     g = f[file_id]\n    #                     print(file_id)\n    #                     for ch, s in enumerate(['H1', 'L1']):\n    #                             x=g[s]['SFTs'].shape\n    #                             print('shape x=',x)\n    #                             try:\n    #                                 a = np.array(g[s]['SFTs'], dtype=np.complex128)\n    #                                 a = a[:360, :first_cut]   # Fourier coefficient complex64\n    #                                 p = 2*(a.real**2 + a.imag**2)  # power\n    #                                 p = normalize_2d(p)\n    #                                 p = np.mean(p.reshape(360, width,compress_1), axis=2)\n    #                             except:\n    #                                 a = np.array(g[s]['SFTs'], dtype=np.complex128)\n    #                                 a = a[:360, :second_cut]  # Fourier coefficient complex64\n    #                                 p = 2*(a.real**2 + a.imag**2)  # power\n    #                                 p = normalize_2d(p)\n    #                                 p = np.mean(p.reshape(360,width,compress_2), axis=2)\n    #                             img[...,ch] = p\n    #     #                 try:\n    #     #                     a=g[\"H1\"]\n    #     #                     a=a['SFTs'][:360, :4320]*1e22\n\n\n    #     #                     #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n    #     #                     q = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n    #     #                     q /= np.mean(q)  # normalize\n    #     #                     q = np.mean(q.reshape(360, 360,12), axis=2)\n    #     #                     print(np.min(q),np.max(q))\n    #     #                     scaler = MinMaxScaler(feature_range=(0, 255))\n    #     #                     scaler = scaler.fit(q)\n    #     #                     img[...,2] = q\n    #     #                     #cv2.imwrite(\"blabla.jpg\",img)\n    #     #                 except:\n    #     #                     a=g[\"H1\"]\n    #     #                     a=a['SFTs'][:360, :4320]*1e22\n    #     #                     p = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n    #     #                     p /= np.mean(p)  # normalize\n    #     #                     p = np.mean(p.reshape(360, 360,11), axis=2)\n    #     #                     scaler = scaler.fit(p)\n    #     #                     img[...,2] = p\n    #             except:\n    #                 print('eroare')\n                    \n            serialized_img = tf.io.serialize_tensor(img)\n            example = serialize_example_train(serialized_img, y, str.encode(file_id))\n            writer.write(example)\n            ","metadata":{"execution":{"iopub.status.busy":"2023-01-02T18:47:04.793852Z","iopub.execute_input":"2023-01-02T18:47:04.794287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T15:32:43.359602Z","iopub.execute_input":"2022-12-18T15:32:43.361006Z","iopub.status.idle":"2022-12-18T15:32:43.380659Z","shell.execute_reply.started":"2022-12-18T15:32:43.360958Z","shell.execute_reply":"2022-12-18T15:32:43.37958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2022-12-18T15:32:43.382347Z","iopub.execute_input":"2022-12-18T15:32:43.382704Z","iopub.status.idle":"2022-12-18T15:32:43.393141Z","shell.execute_reply.started":"2022-12-18T15:32:43.382673Z","shell.execute_reply":"2022-12-18T15:32:43.391983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TEST TF record","metadata":{}},{"cell_type":"code","source":"# test_df=pd.DataFrame()\n# test_df = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\n# #test_files = glob(f\"{ROOT_DIR}/test/*.hdf5\")\n# test_df['cale_fisier']='/kaggle/input/g2net-detecting-continuous-gravitational-waves/test/'+test_df['id']+'.hdf5'\n# # test_df=test_df.sample(10)\n# # test_df=test_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:09:49.652204Z","iopub.execute_input":"2023-01-01T17:09:49.652811Z","iopub.status.idle":"2023-01-01T17:09:49.684778Z","shell.execute_reply.started":"2023-01-01T17:09:49.652778Z","shell.execute_reply":"2023-01-01T17:09:49.683978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:09:50.476911Z","iopub.execute_input":"2023-01-01T17:09:50.477477Z","iopub.status.idle":"2023-01-01T17:09:50.495907Z","shell.execute_reply.started":"2023-01-01T17:09:50.477442Z","shell.execute_reply":"2023-01-01T17:09:50.494947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df=test_df.sample(10)\n# test_df=test_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:32:49.606077Z","iopub.execute_input":"2023-01-01T17:32:49.606446Z","iopub.status.idle":"2023-01-01T17:32:49.614008Z","shell.execute_reply.started":"2023-01-01T17:32:49.60642Z","shell.execute_reply":"2023-01-01T17:32:49.612721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:22:43.54192Z","iopub.execute_input":"2023-01-01T18:22:43.542278Z","iopub.status.idle":"2023-01-01T18:22:43.548408Z","shell.execute_reply.started":"2023-01-01T18:22:43.542241Z","shell.execute_reply":"2023-01-01T18:22:43.547348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #test_df = pd.read_csv(di + '/sample_submission.csv')\n# train_only=False\n# if train_only:\n#     print('skip the nonstationary model dataset save')\n# else:\n#     ct = len(test_df)\n#     idx = test_df.index\n#     f = 0\n#     print(ct)\n#     print('Writing TFRecord %i of %i...'%(f,ct))\n#     xi=0\n#     with tf.io.TFRecordWriter('test%.2i-%i.tfrec'%(f,ct)) as writer:\n#         for _, i in enumerate((tqdm(idx))):\n#             r = test_df.iloc[i]\n#             file_id = r.id\n#             filename=r.cale_fisier\n#             img = np.empty((360, width, 2), dtype=np.float64)\n#             (h1_sfts, h1_ts), (l1_sfts, l1_ts), freq=read_data_c(filename)\n# #             DEBUG=True\n#             #sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)\n#             imagine=sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)\n#             img[...,0] = imagine\n#             imagine=sft_to_img(l1_sfts, l1_ts, buckets=IMG_WIDTH)\n#             img[...,1] = imagine\n#             #filename = '%s/test/%s.hdf5' % (di, file_id)\n# #             with h5py.File(filename, 'r') as f:\n# #                 g = f[file_id]\n# #                 #print('unu')\n# #                 for ch, s in enumerate(['H1', 'L1']):\n# #                     print(xi)\n# #                     xi=xi+1\n                    \n# #                     try:\n# #                         a = np.array(g[s]['SFTs'], dtype=np.complex128)\n# #                         a = a[:360, :first_cut]   # Fourier coefficient complex64\n# #                         p = 2*(a.real**2 + a.imag**2)  # power\n# #                         p = normalize_2d(p)\n# #                         p = np.mean(p.reshape(360, width,compress_1), axis=2)\n# #                     except:\n# #                         a = np.array(g[s]['SFTs'], dtype=np.complex128)\n# #                         a = a[:360, :second_cut]  # Fourier coefficient complex64\n# #                         p = 2*(a.real**2 + a.imag**2)  # power\n# #                         p = normalize_2d(p)\n# #                         p = np.mean(p.reshape(360,width,compress_2), axis=2)\n# #                     img[...,ch] = p\n\n# #     #             try:\n# #     #                 a=g[\"H1\"]\n# #     #                 a=a['SFTs'][:360, :4320]*1e22\n# #     #                 #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n# #     #                 q = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n# #     #                 q /= np.mean(q)  # normalize\n# #     #                 q = np.mean(q.reshape(360, 360,12), axis=2)\n# #     #                 print(np.min(q),np.max(q))\n# #     #                 scaler = MinMaxScaler(feature_range=(0, 255))\n# #     #                 scaler = scaler.fit(q)\n# #     #                 img[...,2] = q\n# #     #                 #cv2.imwrite(\"blabla.jpg\",img)\n# #     #             except:\n# #     #                 a=g[\"H1\"]\n# #     #                 a=a['SFTs'][:360, :3960]*1e22\n# #     #                 p = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n# #     #                 p /= np.mean(p)  # normalize\n# #     #                 p = np.mean(p.reshape(360, 360,11), axis=2)\n# #     #                 scaler = scaler.fit(p)\n# #     #                 img[...,2] = p  \n#             serialized_img = tf.io.serialize_tensor(img)\n#             example = serialize_example_test(serialized_img, str.encode(file_id))\n#             writer.write(example)\n# train_only=False","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:26:59.286113Z","iopub.execute_input":"2023-01-01T18:26:59.286475Z","iopub.status.idle":"2023-01-01T18:27:06.486334Z","shell.execute_reply.started":"2023-01-01T18:26:59.286445Z","shell.execute_reply":"2023-01-01T18:27:06.485329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So simple to get TFREC :)\nThank you!!ITK8191 !","metadata":{}},{"cell_type":"markdown","source":"Load splitted data","metadata":{}},{"cell_type":"markdown","source":"Make a directory","metadata":{}},{"cell_type":"code","source":"#filter_df=pd.read_csv(\"/kaggle/input/flipacoinoncemodel-not-a-eda/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:22.106396Z","iopub.execute_input":"2023-01-01T17:53:22.10706Z","iopub.status.idle":"2023-01-01T17:53:22.111699Z","shell.execute_reply.started":"2023-01-01T17:53:22.107022Z","shell.execute_reply":"2023-01-01T17:53:22.110722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_df = pd.merge(test_df,filter_df, on=['id','id'])","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:22.783018Z","iopub.execute_input":"2023-01-01T17:53:22.783409Z","iopub.status.idle":"2023-01-01T17:53:22.787565Z","shell.execute_reply.started":"2023-01-01T17:53:22.783376Z","shell.execute_reply":"2023-01-01T17:53:22.786684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df=test_df.rename(columns={\"target_y\": \"target\"})\n# test_df=test_df.drop(['target_x'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:23.096975Z","iopub.execute_input":"2023-01-01T17:53:23.097954Z","iopub.status.idle":"2023-01-01T17:53:23.103065Z","shell.execute_reply.started":"2023-01-01T17:53:23.09792Z","shell.execute_reply":"2023-01-01T17:53:23.101006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imagine=sft_to_img(h1_sfts, h1_ts, buckets=IMG_WIDTH)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:58:33.966974Z","iopub.execute_input":"2023-01-01T17:58:33.967579Z","iopub.status.idle":"2023-01-01T17:58:34.010877Z","shell.execute_reply.started":"2023-01-01T17:58:33.96754Z","shell.execute_reply":"2023-01-01T17:58:34.009495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# # img, ts_noise = sft_to_img(h1_sfts, h1_ts)\n# img = sft_to_img(h1_sfts, h1_ts)\n# plt.figure()\n# plt.imshow(img, cmap='viridis')\n","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:24:35.322671Z","iopub.execute_input":"2023-01-01T18:24:35.323155Z","iopub.status.idle":"2023-01-01T18:24:35.488611Z","shell.execute_reply.started":"2023-01-01T18:24:35.323116Z","shell.execute_reply":"2023-01-01T18:24:35.487191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imagine","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:58:48.161469Z","iopub.execute_input":"2023-01-01T17:58:48.161868Z","iopub.status.idle":"2023-01-01T17:58:48.171917Z","shell.execute_reply.started":"2023-01-01T17:58:48.161835Z","shell.execute_reply":"2023-01-01T17:58:48.170635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:23.726417Z","iopub.execute_input":"2023-01-01T17:53:23.727108Z","iopub.status.idle":"2023-01-01T17:53:23.731963Z","shell.execute_reply.started":"2023-01-01T17:53:23.727072Z","shell.execute_reply":"2023-01-01T17:53:23.730722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# sns.displot(data=test_df, x=\"target\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:24.443878Z","iopub.execute_input":"2023-01-01T17:53:24.444231Z","iopub.status.idle":"2023-01-01T17:53:24.447984Z","shell.execute_reply.started":"2023-01-01T17:53:24.444202Z","shell.execute_reply":"2023-01-01T17:53:24.447104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# low_score=test_df[test_df[\"target\"]<=0.68]\n# low_score=low_score.reset_index()\n# high_score=test_df[test_df[\"target\"]>0.68]\n# high_score=high_score.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:24.748578Z","iopub.execute_input":"2023-01-01T17:53:24.749064Z","iopub.status.idle":"2023-01-01T17:53:24.753016Z","shell.execute_reply.started":"2023-01-01T17:53:24.749033Z","shell.execute_reply":"2023-01-01T17:53:24.752112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# low_path = '/kaggle/working/low/'\n# high_path = '/kaggle/working/high/'\n# if not os.path.exists(low_path):\n#     os.mkdir(low_path)\n# if not os.path.exists(high_path):\n#     os.mkdir(high_path)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:25.421174Z","iopub.execute_input":"2023-01-01T17:53:25.42184Z","iopub.status.idle":"2023-01-01T17:53:25.426276Z","shell.execute_reply.started":"2023-01-01T17:53:25.421799Z","shell.execute_reply":"2023-01-01T17:53:25.425174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Low tfrec","metadata":{}},{"cell_type":"code","source":"# if train_only:\n#     print('skip the nonstationary model dataset save')\n# else:\n#     ct = len(low_score)\n#     idx = low_score.index\n#     f = 0\n#     print(ct)\n#     print('Writing TFRecord %i of %i...'%(f,ct))\n#     xi=0\n#     with tf.io.TFRecordWriter('/kaggle/working/low/test%.2i-%i.tfrec'%(f,ct)) as writer:\n#         for _, i in enumerate((tqdm(idx))):\n#             r = test_df.iloc[i]\n#             file_id = r.id\n#             filename=r.cale_fisier\n#             img = np.empty((360, width, 2), dtype=np.float32)\n\n#             #filename = '%s/test/%s.hdf5' % (di, file_id)\n#             with h5py.File(filename, 'r') as f:\n#                 g = f[file_id]\n#                 #print('unu')\n#                 for ch, s in enumerate(['H1', 'L1']):\n#                     print(xi)\n#                     xi=xi+1\n#                     try:\n#                         a = g[s]['SFTs'][:360, :first_cut] * 1e23  # Fourier coefficient complex64\n#                         p = 2*(a.real**2 + a.imag**2)  # power\n#                         p /= np.mean(p)  # normalize\n#                         p = np.mean(p.reshape(360, width,compress_1), axis=2)\n#                         #print('doi')\n#                     except:\n#                         a = g[s]['SFTs'][:360, :second_cut]* 1e23  # Fourier coefficient complex64\n#                         p = 2*(a.real**2 + a.imag**2)  # power\n#                         p /= np.mean(p)  # normalize\n#                         p = np.mean(p.reshape(360, width,compress_2), axis=2)\n#                     img[...,ch] = p\n\n#     #             try:\n#     #                 a=g[\"H1\"]\n#     #                 a=a['SFTs'][:360, :4320]*1e22\n#     #                 #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n#     #                 q = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n#     #                 q /= np.mean(q)  # normalize\n#     #                 q = np.mean(q.reshape(360, 360,12), axis=2)\n#     #                 print(np.min(q),np.max(q))\n#     #                 scaler = MinMaxScaler(feature_range=(0, 255))\n#     #                 scaler = scaler.fit(q)\n#     #                 img[...,2] = q\n#     #                 #cv2.imwrite(\"blabla.jpg\",img)\n#     #             except:\n#     #                 a=g[\"H1\"]\n#     #                 a=a['SFTs'][:360, :3960]*1e22\n#     #                 p = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n#     #                 p /= np.mean(p)  # normalize\n#     #                 p = np.mean(p.reshape(360, 360,11), axis=2)\n#     #                 scaler = scaler.fit(p)\n#     #                 img[...,2] = p  \n#             serialized_img = tf.io.serialize_tensor(img)\n#             example = serialize_example_test(serialized_img, str.encode(file_id))\n#             writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:25.955092Z","iopub.execute_input":"2023-01-01T17:53:25.955486Z","iopub.status.idle":"2023-01-01T17:53:25.961845Z","shell.execute_reply.started":"2023-01-01T17:53:25.955453Z","shell.execute_reply":"2023-01-01T17:53:25.960537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if train_only:\n#     print('skip the nonstationary model dataset save')\n# else:\n#     ct = len(high_score)\n#     idx = high_score.index\n#     f = 0\n#     print(ct)\n#     print('Writing TFRecord %i of %i...'%(f,ct))\n#     xi=0\n#     with tf.io.TFRecordWriter('/kaggle/working/high/test%.2i-%i.tfrec'%(f,ct)) as writer:\n#         for _, i in enumerate((tqdm(idx))):\n#             r = test_df.iloc[i]\n#             file_id = r.id\n#             filename=r.cale_fisier\n#             img = np.empty((360, width, 2), dtype=np.float32)\n\n#             #filename = '%s/test/%s.hdf5' % (di, file_id)\n#             with h5py.File(filename, 'r') as f:\n#                 g = f[file_id]\n#                 #print('unu')\n#                 for ch, s in enumerate(['H1', 'L1']):\n#                     print(xi)\n#                     xi=xi+1\n#                     try:\n#                         a = g[s]['SFTs'][:360, :first_cut] * 1e22  # Fourier coefficient complex64\n#                         p = 2*(a.real**2 + a.imag**2)  # power\n#                         p /= np.mean(p)  # normalize\n#                         p = np.mean(p.reshape(360, width,compress_1), axis=2)\n#                         #print('doi')\n#                     except:\n#                         a = g[s]['SFTs'][:360, :second_cut]* 1e22  # Fourier coefficient complex64\n#                         p = 2*(a.real**2 + a.imag**2)  # power\n#                         p /= np.mean(p)  # normalize\n#                         p = np.mean(p.reshape(360, width,compress_2), axis=2)\n#                     img[...,ch] = p\n\n#     #             try:\n#     #                 a=g[\"H1\"]\n#     #                 a=a['SFTs'][:360, :4320]*1e22\n#     #                 #a = g['H1']['SFTs'][:360, :4320]# Fourier coefficient complex64\n#     #                 q = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n#     #                 q /= np.mean(q)  # normalize\n#     #                 q = np.mean(q.reshape(360, 360,12), axis=2)\n#     #                 print(np.min(q),np.max(q))\n#     #                 scaler = MinMaxScaler(feature_range=(0, 255))\n#     #                 scaler = scaler.fit(q)\n#     #                 img[...,2] = q\n#     #                 #cv2.imwrite(\"blabla.jpg\",img)\n#     #             except:\n#     #                 a=g[\"H1\"]\n#     #                 a=a['SFTs'][:360, :3960]*1e22\n#     #                 p = np.sin(np.angle(a))*(np.abs(a)**2)  # power\n#     #                 p /= np.mean(p)  # normalize\n#     #                 p = np.mean(p.reshape(360, 360,11), axis=2)\n#     #                 scaler = scaler.fit(p)\n#     #                 img[...,2] = p  \n#             serialized_img = tf.io.serialize_tensor(img)\n#             example = serialize_example_test(serialized_img, str.encode(file_id))\n#             writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:53:26.58908Z","iopub.execute_input":"2023-01-01T17:53:26.589471Z","iopub.status.idle":"2023-01-01T17:53:26.597336Z","shell.execute_reply.started":"2023-01-01T17:53:26.58944Z","shell.execute_reply":"2023-01-01T17:53:26.595771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"high tfrec","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}