{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Credits\n\nThis notebook uses [Jun Koda](https://www.kaggle.com/junkoda)'s [spectrogram classification notebook](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification/notebook) as a starter","metadata":{}},{"cell_type":"markdown","source":"# Version\nV1: add custom training dataset to use for pretraining model. Do not train multiple folds on training data as loss/metric is unstable","metadata":{}},{"cell_type":"code","source":"# Use timm pretrained image model\n! pip3 install timm","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:09.119365Z","iopub.execute_input":"2022-11-01T09:07:09.120834Z","iopub.status.idle":"2022-11-01T09:07:21.962411Z","shell.execute_reply.started":"2022-11-01T09:07:09.120658Z","shell.execute_reply":"2022-11-01T09:07:21.961491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport h5py\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport random\nimport gc,os,sys,shutil\n\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom timm.scheduler import CosineLRScheduler\n\nfrom transformers.file_utils import is_torch_tpu_available\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\ncriterion = nn.BCEWithLogitsLoss()\n\n# Train metadata\ndi = '/kaggle/input/g2net-detecting-continuous-gravitational-waves'\ndf = pd.read_csv(di + '/train_labels.csv')\ndf = df[df.target >= 0]  # Remove 3 unknowns (target = -1)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:21.964931Z","iopub.execute_input":"2022-11-01T09:07:21.965619Z","iopub.status.idle":"2022-11-01T09:07:25.782321Z","shell.execute_reply.started":"2022-11-01T09:07:21.96554Z","shell.execute_reply":"2022-11-01T09:07:25.780988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport matplotlib.image\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:25.783519Z","iopub.execute_input":"2022-11-01T09:07:25.783804Z","iopub.status.idle":"2022-11-01T09:07:25.788758Z","shell.execute_reply.started":"2022-11-01T09:07:25.783764Z","shell.execute_reply":"2022-11-01T09:07:25.787772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# params\nPRETRAIN = True\nSINGLE_MODEL = True\nIMG_SIZE = None\nMODEL_NAME = 'tf_efficientnet_b4_ns'\n# MODEL_NAME = 'eca_nfnet_l3'\n# MODEL_NAME = 'levit_192'\n# IMG_SIZE = 224 # resize the images for a specific image models\nN_FOLD = 5\n\n# shapes to resize the images\nSHAPE_1 = (128,32)\nSHAPE_2 = (360,1)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:25.792307Z","iopub.execute_input":"2022-11-01T09:07:25.79435Z","iopub.status.idle":"2022-11-01T09:07:25.837237Z","shell.execute_reply.started":"2022-11-01T09:07:25.79428Z","shell.execute_reply":"2022-11-01T09:07:25.835875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:25.838691Z","iopub.execute_input":"2022-11-01T09:07:25.839128Z","iopub.status.idle":"2022-11-01T09:07:25.852981Z","shell.execute_reply.started":"2022-11-01T09:07:25.839094Z","shell.execute_reply":"2022-11-01T09:07:25.852067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class ZipDataset(torch.utils.data.Dataset):\n    \"\"\"\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 180\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, pos_len=5000, neg_len=9900, path='../input/g2net-generated-data-1/',mod= 100, noise = 0.99):\n        self.path = path\n        self.mod = int(mod)\n        self.noise = noise\n        self.noise_type = 2\n        \n        self.pos_len = pos_len\n        self.neg_len = neg_len\n        self.len = pos_len + neg_len\n        \n        self.mixup = True\n        self.mixup_prob = 0.1\n        self.perm_pos = np.random.permutation(np.arange(self.pos_len))\n        self.perm_neg = np.random.permutation(np.arange(self.neg_len))\n    def __len__(self):\n        return self.len\n    def gen_noise(self, shape):\n        ns = 0.15\n        nr = 0.05\n\n        noise_shape = (360*4140)\n        noise_L_r = self.gen_noise(noise_shape)\n        noise_H_r = noise_L_r*(1-ns)+self.gen_noise(noise_shape)*ns\n        noise_L_i = noise_L_r*(1-nr)+self.gen_noise(noise_shape)*nr\n        noise_H_i = noise_H_r*(1-nr)+self.gen_noise(noise_shape)*nr\n\n        noise_r = np.stack([noise_L_r,noise_H_r]) *1e22\n        noise_i = np.stack([noise_L_i,noise_H_i]) *1e22\n        img_n = noise_r**2 + noise_i**2\n        return img_n\n    def get_negative(self,i):\n        file_name = f'../input/g2net-generated-signals/archive/0_data_{self.mod*(1+(i)//self.mod)}/signals_{i%self.mod}.npy'\n        img = np.load(file_name).astype(np.float64)\n        y=0.0\n        return img, y\n    def get_positive(self, i):\n        file_name = f'../input/g2net-generated-signals/archive/1_data_{self.mod*(1+(i)//self.mod)}/signals_{i%self.mod}.npy'\n        img = np.load(file_name).astype(np.float64)\n        \n        noise_id = int(random.random()*self.neg_len)\n        noise_r = random.random()*0.05+0.95\n        img = (np.sqrt(img)*(1-noise_r)+np.sqrt(self.get_negative(noise_id)[0])*noise_r)**2\n        y=1.0\n        return img, y\n    def get_noise(self):\n        return self.noise\n    def get_mixup(self, i, t):\n        if t==1:\n            mix_img = (self.get_positive(i)[0] + self.get_positive(self.perm_pos[i])[0])/2\n            if random.random() < 1/self.pos_len:\n                self.pos_perm = np.random.permutation(np.arange(self.pos_len))\n        else:\n            mix_img = (self.get_negative(i)[0] + self.get_negative(self.perm_neg[i])[0])/2\n            if random.random() < 1/self.neg_len:\n                self.neg_perm = np.random.permutation(np.arange(self.neg_len))\n        return mix_img, t\n    def __getitem__(self, i):\n        if i<self.pos_len:\n            if self.mixup and random.random() < self.mixup_prob:\n                img, y = self.get_mixup(i,1)\n            else:\n                img, y = self.get_positive(i)\n        else:\n            i = i-self.pos_len\n            if self.mixup and random.random() < self.mixup_prob:\n                img, y = self.get_mixup(i,0)\n            else:\n                img, y = self.get_negative(i)\n        img = ((img)/img.mean() ).astype(np.float32)\n        return img, y\n    \nclass H5Dataset(torch.utils.data.Dataset):\n    \"\"\"\n    dataset = Dataset(data_type, df)\n\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 128\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self, data_type, df):\n        self.data_type = data_type\n        self.df = df\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        \"\"\"\n        i (int): get ith data\n        \"\"\"\n        r = self.df.iloc[i]\n        y = np.float32(r.target)\n        file_id = r.id\n\n        img = np.empty((2, SHAPE_2[0], SHAPE_1[0]), dtype=np.float32)\n\n        filename = '%s/%s/%s.hdf5' % (di, self.data_type, file_id)\n        with h5py.File(filename, 'r') as f:\n            g = f[file_id]\n\n            for ch, s in enumerate(['H1', 'L1']):\n                a = g[s]['SFTs'][:, :SHAPE_1[0]*SHAPE_1[1]] * 1e22  # Fourier coefficient complex64\n\n                p = a.real**2 + a.imag**2  # power\n                p = np.mean(p.reshape(360, SHAPE_1[0],SHAPE_1[1]), axis=2)\n                p = np.mean(p.reshape(SHAPE_2[0],SHAPE_2[1],SHAPE_1[0]), axis=1)\n\n                img[ch] = p\n        img = ((img)/img.mean()).astype(np.float32)\n        return img, y","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:25.854415Z","iopub.execute_input":"2022-11-01T09:07:25.85499Z","iopub.status.idle":"2022-11-01T09:07:25.881971Z","shell.execute_reply.started":"2022-11-01T09:07:25.854931Z","shell.execute_reply":"2022-11-01T09:07:25.881032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/g2net-detecting-continuous-gravitational-waves'\nos.path.isdir(ROOT_DIR)\n\nstr_path4=\"/kaggle/working/spectrogram-images/\"\npath4 = Path(str_path4)\nstr_path3=\"/kaggle/working/spectrogram-images/train/\"\npath3 = Path(str_path3)\nstr_path1=\"/kaggle/working/spectrogram-images/train/signal\"\npath1 = Path(str_path1)\nstr_path2=\"/kaggle/working/spectrogram-images/test\"\npath2 = Path(str_path2)\nstr_path0=\"/kaggle/working/spectrogram-images/train/noise\"\npath0 = Path(str_path0)\n\n\nif not os.path.exists(path4):\n    os.mkdir(path4)\nprint(path4)\nif not os.path.exists(path3):\n    os.mkdir(path3)\nprint(path3)\nif not os.path.exists(path0):\n    os.mkdir(path0)\nif not os.path.exists(path1):\n    os.mkdir(path1)\nif not os.path.exists(path2):\n    os.mkdir(path2)\n\ndef calea(label):\n    if label == 0:\n        return path0\n    else:\n        return path1","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:25.88352Z","iopub.execute_input":"2022-11-01T09:07:25.884502Z","iopub.status.idle":"2022-11-01T09:07:25.904618Z","shell.execute_reply.started":"2022-11-01T09:07:25.884461Z","shell.execute_reply":"2022-11-01T09:07:25.903308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize data","metadata":{}},{"cell_type":"code","source":"# dataset = H5Dataset('train', df)\n# img_real, y = dataset[1]\ndataset = ZipDataset()\nimg_label0_1, y = dataset[4900]\nprint(y)\nimg_label0_2, y = dataset[5901]\nprint(y)\nimg_label1_1, y = dataset[2]\nprint(y)\nimg_label1_2, y = dataset[4]\nprint(y)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:25.906217Z","iopub.execute_input":"2022-11-01T09:07:25.907049Z","iopub.status.idle":"2022-11-01T09:07:26.023194Z","shell.execute_reply.started":"2022-11-01T09:07:25.907012Z","shell.execute_reply":"2022-11-01T09:07:26.022031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setari","metadata":{"execution":{"iopub.status.busy":"2022-10-31T14:06:48.427226Z","iopub.execute_input":"2022-10-31T14:06:48.427702Z","iopub.status.idle":"2022-10-31T14:06:48.433847Z","shell.execute_reply.started":"2022-10-31T14:06:48.427668Z","shell.execute_reply":"2022-10-31T14:06:48.432185Z"}}},{"cell_type":"code","source":"paleta_culori='BuPu'\nSHAPE_1 = (180,23)\nSHAPE_2 = (360,1)\ntrimsize=SHAPE_1[0]*SHAPE_1[1]\ntrimsize","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:26.024928Z","iopub.execute_input":"2022-11-01T09:07:26.025339Z","iopub.status.idle":"2022-11-01T09:07:26.032814Z","shell.execute_reply.started":"2022-11-01T09:07:26.025306Z","shell.execute_reply":"2022-11-01T09:07:26.031856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor i in range(len(dataset)):\n# for i in range(3):\n    img, y = dataset[i]\n    #lista_etichete.append(y)\n    print(img[0].shape)\n    print(calea(y))\n    cale=calea(y)\n    numefisier0=str(i)+\"-h_simulated_power.jpg\"\n    file_path_0 = os.path.join(cale,numefisier0)\n    print(file_path_0)\n    matplotlib.image.imsave( file_path_0, img[0],cmap=paleta_culori)\n    numefisier1=str(i)+\"-l_simulated_power.jpg\"\n    file_path_0 = os.path.join(cale,numefisier1)\n    matplotlib.image.imsave( file_path_0, img[1],cmap=paleta_culori)\n\n#lista1=pd.DataFrame(lista_etichete)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:26.034696Z","iopub.execute_input":"2022-11-01T09:07:26.035075Z","iopub.status.idle":"2022-11-01T09:07:26.151379Z","shell.execute_reply.started":"2022-11-01T09:07:26.035043Z","shell.execute_reply":"2022-11-01T09:07:26.149939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The rest of the show","metadata":{}},{"cell_type":"code","source":"import h5py\nfrom glob import glob","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:26.155941Z","iopub.execute_input":"2022-11-01T09:07:26.1563Z","iopub.status.idle":"2022-11-01T09:07:26.161709Z","shell.execute_reply.started":"2022-11-01T09:07:26.156272Z","shell.execute_reply":"2022-11-01T09:07:26.16034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(f\"{ROOT_DIR}/train_labels.csv\")\ntest = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\ntrain_files = glob(f\"{ROOT_DIR}/train/*.hdf5\")\ntest_files = glob(f\"{ROOT_DIR}/test/*.hdf5\")\ntrain_files[:5]","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:26.163099Z","iopub.execute_input":"2022-11-01T09:07:26.163389Z","iopub.status.idle":"2022-11-01T09:07:26.420057Z","shell.execute_reply.started":"2022-11-01T09:07:26.163358Z","shell.execute_reply":"2022-11-01T09:07:26.418973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Utility to read hdf5 file","metadata":{}},{"cell_type":"markdown","source":"Utility to read hdf5 file","metadata":{}},{"cell_type":"code","source":"# Utility to read hdf5 file\ndef read_data(file: Path):\n    with h5py.File(file, \"r\") as f:\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        freq_hz = list(f[\"frequency_Hz\"])\n        ###\n        h1_stft = h1[\"SFTs\"][()]\n        ###\n        h1_timestamp = h1[\"timestamps_GPS\"][()]\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        l1_timestamp = l1[\"timestamps_GPS\"][()]\n        \n        return {\n            \"H1\": [h1_stft, h1_timestamp],\n            \"L1\": [l1_stft, l1_timestamp],\n            \"freq_hz\": freq_hz\n        }","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:26.421071Z","iopub.execute_input":"2022-11-01T09:07:26.421428Z","iopub.status.idle":"2022-11-01T09:07:26.428813Z","shell.execute_reply.started":"2022-11-01T09:07:26.421398Z","shell.execute_reply":"2022-11-01T09:07:26.427926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utilities","metadata":{}},{"cell_type":"markdown","source":"># shapes to resize the images\n> SHAPE_1 = (128,32)\n> SHAPE_2 = (360,1) \n\n> a = g[s]['SFTs'][:, :SHAPE_1[0]*SHAPE_1[1]] * 1e22  # Fourier coefficient complex64\n> \n>                 p = a.real**2 + a.imag**2  # power\n>                 p = np.mean(p.reshape(360, 128,32, axis=2)\n                 # example p.mean(p.reshape(360, SHAPE_1[0],SHAPE_1[1]), axis=2)\n>                 p = np.mean(p.reshape(360,1,128, axis=1)\n> \n>                 img[ch] = p\n>         img = ((img)/img.mean()).astype(np.float32)","metadata":{}},{"cell_type":"code","source":"# SHAPE_1 = (128,32)\n# SHAPE_2 = (360,1)\n\ndef power_spectrogram_train(h1_sft,cale,z,ind):\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    img = np.empty((1,SHAPE_2[0],SHAPE_1[0]), dtype=np.float32)\n    a=h1_sft[:, :SHAPE_1[0]*SHAPE_1[1]] * 1e22\n    p = 2*(a.real**2 + a.imag**2)/(1800 * 1e24** 2) \n    p = a.real**2 + a.imag**2  # power\n    p /= np.mean(p)  # normalize\n    p = np.mean(p.reshape(360, SHAPE_1[0],SHAPE_1[1]), axis=2)\n    img[0] = p\n    img=img[0]\n    img=((img)/img.mean()).astype(np.float32)\n    numefisier=ind+z+\"train_power.jpg\"\n    file_path_1 = os.path.join(cale,numefisier)\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    matplotlib.image.imsave( file_path_1, img,cmap=paleta_culori)\n#     plt.figure(figsize = (20,10))\n#     plt.imshow(img[0], interpolation = 'sinc')\ndef power_spectrogram_augumented(h1_sft,cale,z,ind):\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    img = np.empty((2,SHAPE_2[0],SHAPE_1[0]), dtype=np.float32)\n    a=h1_sft[:, :SHAPE_1[0]*SHAPE_1[1]] * 1e22\n    p = 2*(a.real**2 + a.imag**2)/(1800 * 1e24** 2) \n    p = a.real**2 + a.imag**2  # power\n    p /= np.mean(p)  # normalize\n    p = p.reshape(360, SHAPE_1[0],SHAPE_1[1])\n#     praw=p\n#     praw\n    #print(p.shape)\n    praw=np.mean(p[:,:,0:16],axis=2)\n    img[0] = praw\n    img1=img[0]\n    img2=((img1)/img1.mean()).astype(np.float32)\n    numefisier=ind+z+\"_augumented_train_1_power.jpg\"\n    file_path_1 = os.path.join(cale,numefisier)\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    matplotlib.image.imsave( file_path_1, img2,cmap=paleta_culori)\n    #print(p.shape)\n    praw1=np.mean(p[:,:,10:23],axis=2)\n    img[1] = praw1\n    img3=img[1]\n    img4=((img3)/img3.mean()).astype(np.float32)\n    numefisier=ind+z+\"_augumented_train_2_power.jpg\"\n    file_path_2 = os.path.join(cale,numefisier)\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    matplotlib.image.imsave( file_path_2, img4,cmap=paleta_culori)\n    praw1=np.mean(p[:,:,6:15],axis=2)\n    img[1] = praw1\n    img3=img[1]\n    img4=((img3)/img3.mean()).astype(np.float32)\n    numefisier=ind+z+\"_augumented_train_3_power.jpg\"\n    file_path_2 = os.path.join(cale,numefisier)\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    matplotlib.image.imsave( file_path_2, img4,cmap=paleta_culori)\n    \n    \n#     plt.figure(figsize = (20,10))\n#     plt.imshow(img[0], interpolation = 'sinc')    \ndef power_spectrogram_test(h1_sft,cale,z,ind):\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    img = np.empty((1,SHAPE_2[0],SHAPE_1[0]), dtype=np.float32)\n    a=h1_sft[:, :SHAPE_1[0]*SHAPE_1[1]] * 1e22\n    p = 2*(a.real**2 + a.imag**2)/(1800 * 1e24** 2) \n    p = a.real**2 + a.imag**2  # power\n    p /= np.mean(p)  # normalize\n    p = np.mean(p.reshape(360, SHAPE_1[0],SHAPE_1[1]), axis=2)\n    img[0] = p\n    img=img[0]\n    img=((img)/img.mean()).astype(np.float32)\n    numefisier=ind+z+\"test_power.jpg\"\n    file_path_1 = os.path.join(cale,numefisier)\n    #matplotlib.rcParams['image.cmap'] = 'viridis'\n    matplotlib.image.imsave( file_path_1, img,cmap=paleta_culori)\n#     plt.figure(figsize = (20,10))\n#     plt.imshow(img[0], interpolation = 'sinc')   ","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:23:07.130674Z","iopub.execute_input":"2022-11-01T09:23:07.131076Z","iopub.status.idle":"2022-11-01T09:23:07.152957Z","shell.execute_reply.started":"2022-11-01T09:23:07.131042Z","shell.execute_reply":"2022-11-01T09:23:07.151677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Augument data","metadata":{}},{"cell_type":"code","source":"# def get_sft(i):\n#     fisier = Path(train_files[i])\n#     data = read_data(fisier)\n#     h1_sft, h1_ts = data[\"H1\"]\n#     l1_sft, l1_ts = data[\"L1\"]\n#     freq_hz = data[\"freq_hz\"]\n#     #label = train_labels.loc[train_labels[\"id\"] == ind][\"target\"].values[0]\n#     return h1_sft,l1_sft","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:13:29.795186Z","iopub.execute_input":"2022-11-01T09:13:29.795565Z","iopub.status.idle":"2022-11-01T09:13:29.802771Z","shell.execute_reply.started":"2022-11-01T09:13:29.795537Z","shell.execute_reply":"2022-11-01T09:13:29.80198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# h1_sft,l1_sft=get_sft(1)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:31:11.202394Z","iopub.execute_input":"2022-11-01T09:31:11.203751Z","iopub.status.idle":"2022-11-01T09:31:11.208404Z","shell.execute_reply.started":"2022-11-01T09:31:11.203688Z","shell.execute_reply":"2022-11-01T09:31:11.207292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #matplotlib.rcParams['image.cmap'] = 'viridis'\n# img = np.empty((2,SHAPE_2[0],SHAPE_1[0]), dtype=np.float32)\n# a=h1_sft[:, :SHAPE_1[0]*SHAPE_1[1]] * 1e22\n# p = 2*(a.real**2 + a.imag**2)/(1800 * 1e24** 2) \n# p = a.real**2 + a.imag**2  # power\n# p /= np.mean(p)  # normalize\n# p = p.reshape(360, SHAPE_1[0],SHAPE_1[1])\n# #     praw=p\n# #     praw\n# #print(p.shape)\n# praw=np.mean(p[:,:,10:23],axis=2)\n# img[0] = praw\n# img1=img[0]\n# img2=((img1)/img1.mean()).astype(np.float32)\n# plt.imshow(img2)\n# #numefisier=ind+z+\"_augumented_train_1_power.jpg\"\n# #file_path_1 = os.path.join(cale,numefisier)\n# #matplotlib.rcParams['image.cmap'] = 'viridis'\n# #matplotlib.image.imsave( file_path_1, img2,cmap=paleta_culori)\n# print(p.shape)\n# #praw1=\n# praw1=np.mean(p[:,:,0:13],axis=2)\n# print(praw1.shape)\n# print(praw1)\n# img[1] = praw1\n# img3=img[1]\n# img4=((img3)/img3.mean()).astype(np.float32)\n# # plt.imshow(img4)\n# #numefisier=ind+z+\"_augumented_train_2_power.jpg\"\n# #file_path_2 = os.path.join(cale,numefisier)\n# #matplotlib.rcParams['image.cmap'] = 'viridis'\n# #matplotlib.image.imsave( file_path_2, img4,cmap=paleta_culori)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:31:11.834744Z","iopub.execute_input":"2022-11-01T09:31:11.835188Z","iopub.status.idle":"2022-11-01T09:31:11.841365Z","shell.execute_reply.started":"2022-11-01T09:31:11.835152Z","shell.execute_reply":"2022-11-01T09:31:11.840191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train","metadata":{}},{"cell_type":"code","source":"# OUTPUT_DIR = '/kaggle/working/'\n# dist_dir = os.path.join(OUTPUT_DIR, '1')\n\ndef dataloader(filenames):\n#     matricea = []\n#     matricea2 = []\n    contor =0\n   \n    for filename in filenames:\n#                  if contor <3:\n                    with h5py.File(filename, \"r\") as f:\n                        # check if label is valid\n                        ind, *_ = f.keys()\n                        #print(f.keys())\n                        label = train_labels.loc[train_labels[\"id\"] == ind][\"target\"].values[0]\n                        if label == -1:\n                            #print(\"found one\")\n                            #print(ind)\n                            continue\n                        #print(\"eticheta este\",label)\n                        cale=calea(label)\n                        #print(cale)\n                        #print(\"debug0\")\n                        f = f[ind]\n                        #print(ind)\n                        h1 = f[\"H1\"]\n                        #print(h1)\n                        l1 = f[\"L1\"]\n                        freq_hz = list(f[\"frequency_Hz\"])\n                        h1_sft = h1[\"SFTs\"][()]\n                        #print\n                        h1_timestamp = h1[\"timestamps_GPS\"][()]\n            #         # H2 data\n                        l1_sft = l1[\"SFTs\"][()]\n                        l1_timestamp = l1[\"timestamps_GPS\"][()]\n                        #dector name\n                        #print(cale)\n                        z=\"_h\"\n                        #print(z)\n                        #imageshape320(h1_sft,cale,z,ind,label)\n                        power_spectrogram_train(h1_sft,cale,z,ind)\n                        power_spectrogram_augumented(h1_sft,cale,z,ind)\n                        #power_spectrogram(h1_stft,cale,z,ind)\n                                                #dector name\n                        z=\"_l\"\n                        #print(z)\n                        #imageshape320(l1_sft,cale,z,ind,label)\n                        power_spectrogram_train(l1_sft,cale,z,ind)\n                        power_spectrogram_augumented(l1_sft,cale,z,ind)\n                        contor=contor+1\n                        #print(contor)\n\n    return contor#,pd.DataFrame(matricea2)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:22:13.196236Z","iopub.execute_input":"2022-11-01T09:22:13.196597Z","iopub.status.idle":"2022-11-01T09:22:13.20758Z","shell.execute_reply.started":"2022-11-01T09:22:13.19657Z","shell.execute_reply":"2022-11-01T09:22:13.206171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_h1=dataloader(train_files)\n# print(\"files:\",train_h1)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:22:19.144429Z","iopub.execute_input":"2022-11-01T09:22:19.144785Z","iopub.status.idle":"2022-11-01T09:22:20.529921Z","shell.execute_reply.started":"2022-11-01T09:22:19.14476Z","shell.execute_reply":"2022-11-01T09:22:20.528784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test","metadata":{}},{"cell_type":"code","source":"def dataloader2(filenames):\n    matricea = []\n#     matricea2 = []\n    contor =0\n   \n    for filename in filenames:\n#                  if contor <=3:\n                    with h5py.File(filename, \"r\") as f:\n                        ind, *_ = f.keys()\n                        cale=path2\n                        f = f[ind]\n                        h1 = f[\"H1\"]\n                        l1 = f[\"L1\"]\n                        freq_hz = list(f[\"frequency_Hz\"])\n                        h1_sft = h1[\"SFTs\"][()]\n                        h1_timestamp = h1[\"timestamps_GPS\"][()]\n                        l1_sft = l1[\"SFTs\"][()]\n                        l1_timestamp = l1[\"timestamps_GPS\"][()]\n                        if h1_sft.shape[1] <trimsize and l1_stft.shape[1] <trimsize:\n                                    print('ind')  \n                        else:\n                                    #detector name\n                                    z=\"_h\"\n                                    #print(z)\n                                    #power_spectrogram360(h1_stft,cale,z,ind)\n                                    power_spectrogram_test(h1_sft,cale,z,ind)\n                                    #dector name\n                                    z=\"_l\"\n                                    #print(z)\n                                    #power_spectrogram360(l1_stft,cale,z,ind)\n                                    power_spectrogram_test(l1_sft,cale,z,ind)\n                        contor=contor+1\n                        #print(contor)\n#     print(contor)\n#     return matricea","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:27.522866Z","iopub.execute_input":"2022-11-01T09:07:27.523658Z","iopub.status.idle":"2022-11-01T09:07:27.532135Z","shell.execute_reply.started":"2022-11-01T09:07:27.523613Z","shell.execute_reply":"2022-11-01T09:07:27.531034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndataloader2(test_files)","metadata":{"execution":{"iopub.status.busy":"2022-11-01T09:07:27.533123Z","iopub.execute_input":"2022-11-01T09:07:27.53366Z","iopub.status.idle":"2022-11-01T09:07:28.788175Z","shell.execute_reply.started":"2022-11-01T09:07:27.533634Z","shell.execute_reply":"2022-11-01T09:07:28.786883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}