{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torchvision==0.13.1\n!pip install torchinfo\n! pip uninstall torchmetrics --y","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:12:10.423302Z","iopub.execute_input":"2023-06-14T17:12:10.423969Z","iopub.status.idle":"2023-06-14T17:13:43.273201Z","shell.execute_reply.started":"2023-06-14T17:12:10.423925Z","shell.execute_reply":"2023-06-14T17:13:43.271966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchinfo","metadata":{"executionInfo":{"elapsed":4571,"status":"ok","timestamp":1669050403527,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"NLiwS4-TZrqp","outputId":"961eb75d-e35e-42d4-8f05-56b6ad8083ea","execution":{"iopub.status.busy":"2023-06-14T17:13:43.276309Z","iopub.execute_input":"2023-06-14T17:13:43.276955Z","iopub.status.idle":"2023-06-14T17:13:54.779397Z","shell.execute_reply.started":"2023-06-14T17:13:43.276909Z","shell.execute_reply":"2023-06-14T17:13:54.778221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip uninstall torchmetrics --y","metadata":{"executionInfo":{"elapsed":3866,"status":"ok","timestamp":1669050407353,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"w9cGT1JG73Qx","outputId":"69289cd6-f347-49de-da09-f25ceeaff452","execution":{"iopub.status.busy":"2023-06-14T17:13:54.781543Z","iopub.execute_input":"2023-06-14T17:13:54.781954Z","iopub.status.idle":"2023-06-14T17:13:56.858333Z","shell.execute_reply.started":"2023-06-14T17:13:54.781905Z","shell.execute_reply":"2023-06-14T17:13:56.85713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install torchmetrics","metadata":{"execution":{"iopub.status.busy":"2023-06-14T17:14:22.16923Z","iopub.execute_input":"2023-06-14T17:14:22.169646Z","iopub.status.idle":"2023-06-14T17:14:34.777874Z","shell.execute_reply.started":"2023-06-14T17:14:22.169607Z","shell.execute_reply":"2023-06-14T17:14:34.776543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom glob import glob\nfrom tqdm import tqdm\nimport time\nfrom datetime import timedelta\nfrom PIL import Image, ImageFilter\nimport cv2\nimport scipy.ndimage\nimport os\nfrom torchmetrics.classification import BinaryAUROC\n\nfrom ipywidgets import interact, fixed\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nimport torchvision\nfrom torchinfo import summary\nfrom torchvision.models.feature_extraction import create_feature_extractor\nfrom torchvision.transforms.functional import InterpolationMode\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport time\nimport h5py\n# import timm\nimport torch\nimport torch.nn as nn\nimport random\nimport psutil\n\nrandom.seed(42)\ntorch.manual_seed(42)\nnp.random.seed(42)\n\nimport datetime\n\ntime_now  = datetime.datetime.now().strftime('%d_%m_%Y-%H_%M_%S') \nprint(time_now)","metadata":{"executionInfo":{"elapsed":3454,"status":"ok","timestamp":1669050410800,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"3qCrB9gAZrts","execution":{"iopub.status.busy":"2023-06-14T17:14:34.780506Z","iopub.execute_input":"2023-06-14T17:14:34.780947Z","iopub.status.idle":"2023-06-14T17:14:37.934787Z","shell.execute_reply.started":"2023-06-14T17:14:34.780903Z","shell.execute_reply":"2023-06-14T17:14:37.933416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1669050411624,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"0B2aUmeO1kHR","execution":{"iopub.status.busy":"2023-06-14T17:14:37.938113Z","iopub.execute_input":"2023-06-14T17:14:37.939042Z","iopub.status.idle":"2023-06-14T17:14:38.043593Z","shell.execute_reply.started":"2023-06-14T17:14:37.939004Z","shell.execute_reply":"2023-06-14T17:14:38.042184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = {}\nCFG['MODEL_WEIGHTS_PATH'] = \"/kaggle/working/\"\n# CFG['DATASET']            = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves\"\nCFG['DATASET']            = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves\"\nCFG['TRAIN_PATH']         = \"/kaggle/input/360x256-uint8/train_full_360x256_uint8/train_full_360x256_uint8\"\nCFG['VALID_PATH']         = \"/kaggle/input/360x256-uint8/valid_full_360x256_uint8/valid_full_360x256_uint8\"\n\nCFG[\"SAMPLE_FEATURES\"]    = 1\nCFG[\"INPUT_SHAPE\"]        = (2,360,256)  \nCFG[\"BATCH_SIZE\"]         = 32\n                            \n\nCFG[\"SIGNAL_LVL\"]         = [0.03, 0.15]\n\nCFG['NUM_EPOCHS']         = 150\n\nCFG['WEIGHTS_NAME']        = f\"{time_now}_BEST_{CFG['SIGNAL_LVL'][0]}-{CFG['SIGNAL_LVL'][1]}\"\nCFG['WEIGHTS_NAME_BACKUP'] = f\"{time_now}_BACKUP_{CFG['SIGNAL_LVL'][0]}-{CFG['SIGNAL_LVL'][1]}\"","metadata":{"executionInfo":{"elapsed":274,"status":"ok","timestamp":1669050936606,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"z7GANr7xZeO4","execution":{"iopub.status.busy":"2023-06-14T17:14:38.046777Z","iopub.execute_input":"2023-06-14T17:14:38.047459Z","iopub.status.idle":"2023-06-14T17:14:38.058584Z","shell.execute_reply.started":"2023-06-14T17:14:38.047416Z","shell.execute_reply":"2023-06-14T17:14:38.057663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class G2Sample():\n    def __init__(    self, \n                     file_idx=None, \n                     hdf=None, \n                     idx=None, \n                     label=None, \n                     h1=None, \n                     l1=None, \n                     loaded=False, \n                     valid=False,\n                ):\n        \n        self.hdf       = hdf\n        self.file_idx  = file_idx\n        self.idx       = idx\n        self.label     = label\n        self.h1        = h1 \n        self.l1        = l1\n        self.valid     = valid\n        self.loaded    = loaded\n        \n\n    def preload(self):\n        file = self.hdf\n        with h5py.File(file, \"r\") as f:\n            f = f[self.file_idx]\n            h1 = f[\"H1\"]\n            l1 = f[\"L1\"]\n        \n#             h1_stft = h1[\"SFTs\"][()] * 1e22\n#             h1_img = self.transform_img_part(h1_stft)\n#             h1_img /= np.mean(h1_img)                    #normalize  -> mean = 1\n#             self.h1 = h1_img\n\n            l1_stft = l1[\"SFTs\"][()] * 1e22\n            l1_img = self.transform_img_part(l1_stft)\n            l1_img /= np.mean(l1_img)                    #normalize  -> mean = 1\n            self.l1 = l1_img\n            self.loaded = True\n    \n    \n    def unload(self):\n        self.h1 = None \n        self.l1 = None\n        self.loaded = False\n        \n        \n    def transform_img_part(self, img_original):\n        return img_original.real**2 + img_original.imag**2\n    \n    \n\nclass G2Dataset(torch.utils.data.Dataset):\n    def __init__(self, root_dir, preload=True, fake_train_numb=3200, segmentation_mode=False):\n        self.root_dir = root_dir\n        self.fake_train_numb = fake_train_numb\n        self.segmentation_mode = segmentation_mode\n        self.__samples = []\n        self.train = True\n        self.preload = preload\n        if preload:\n            self.__preload()\n        else:\n            self.__load()\n        self.s1 = 0\n        self.s2 = 0\n        \n        \n    def __get_train_idx(self, mode):\n        labels = []\n        for i in range(len(self.__samples)):\n            if mode == \"Train\":\n                valid = self.__samples[i].valid\n                idx = self.__samples[i].idx\n                if not valid and idx:\n                    labels.append(i)\n            if mode == \"Valid\":\n                valid = self.__samples[i].valid\n                if valid:\n                    labels.append(i)\n        return labels\n        \n        \n    def __preload(self):\n        file_name_list = os.listdir(f\"{CFG['DATASET']}/test/\")\n        for i, file_name in tqdm(enumerate(file_name_list)):\n            file_idx = file_name.split('.')[0]\n            sample = G2Sample(idx=i, \n                              hdf=f\"{CFG['DATASET']}/test/{file_name}\",\n                              file_idx=file_idx)\n            self.__samples.append(sample)\n    \n    \n    def __load(self):\n        #load valid samples\n        df = pd.read_csv(self.root_dir + '/train_labels.csv')\n        df = df[df.target >= 0]  # Remove 3 unknowns (target = -1)\n        for i, row in tqdm(df.iterrows()):\n            idx = row[\"id\"]\n            target = row[\"target\"]\n            if idx in ['4f6fb7aec', '62a276084', 'b72691822']:\n                continue\n            self.__read_data(idx, target, valid=True)\n#             if i == 25:\n#                 break\n        print(f\"Successfully loaded {i} validation samples\")\n        \n        \n        #load train samples\n        arr = os.listdir(CFG['TRAIN_PATH'])\n        for i, file_name in tqdm(enumerate(arr)):\n            idx = file_name.split('.')[0]\n            target = None \n            self.__read_data(idx, target, valid=False)\n#             if i == 25:\n#                 break\n        print(f\"Successfully loaded {i} train samples\")\n        \n        \n\n\n        \n    def __read_data(self, idx, label, valid):\n        if valid:\n            file=f\"{CFG['VALID_PATH']}/{idx}.npz\"\n        else:\n            file=f\"{CFG['TRAIN_PATH']}/{idx}.npz\"\n        h1l1 = np.load(file)\n        h1_img = (h1l1[\"H1\"]/255.).astype(np.float16)\n        l1_img = (h1l1[\"L1\"]/255.).astype(np.float16)\n        sample = G2Sample(idx=idx, \n                          label=label, \n                          h1=h1_img, \n                          l1=l1_img,\n                          loaded = True,\n                          valid = valid)\n        self.__samples.append(sample)\n\n\n    def __len__(self):\n        return len(self.__samples)\n    \n    \n    def at(self, idx):\n        sample = self.__samples[idx]\n        return sample\n        \n        \n    def __transform_img_part(self, img_original):\n        return img_original.real**2 + img_original.imag**2\n    \n    \n    def __preprocess(self, im1,im2): \n\n        im1 = torch.tensor(im1).to(device)\n        im2 = torch.tensor(im2).to(device)\n        \n        im1 = im1.unsqueeze(dim=0)\n        im2 = im2.unsqueeze(dim=0)\n\n        \n        x = torch.cat([im1, im2], dim=0)\n        normalize = transforms.Compose([    \n            transforms.Lambda(lambda x: (x - x.min())/((x.max() - x.min()))),\n        ])\n\n        x = normalize(x)\n        x = x.cpu().numpy().astype(np.float32)\n        return x\n    \n    \n    def __reshape(self, img, final_width):\n        img = img.squeeze()\n        img = img[:, :4096]\n        time_mean = int(4096 / final_width)\n        img = np.mean(img.reshape(360, int(img.shape[1]/time_mean), time_mean), axis=2)\n#         img = np.expand_dims(img, axis=0)\n        return img\n    \n    \n    def __gen_noise_by_shuffle(self, img):\n        h, w = img.shape[0], img.shape[1]\n        np.random.shuffle(img.ravel())\n        img = np.reshape(img, (h, w))\n        return img\n\n    \n    def __gen_noise_by_shuffle_each_column(self, img):\n        for i in range(img.shape[1]):\n            np.random.shuffle(img[:,i])\n        return img\n    \n    \n    def __generate_sin(self, x_max=125 ):\n        x = np.linspace(0, x_max, x_max*10)\n        mult = np.random.choice(range(2, 10))/10\n        shift = np.random.choice(range(x_max*6))\n        def_freq = ((x*mult)+shift)/x_max\n        y = np.sin(def_freq)*180+180\n        #shift on y axis\n        shift_space = y.min() if y.min() > 360 - y.max() else 360 - y.max()\n        shift_space = np.random.choice(range(int(shift_space)))\n        if y.min() > 360 - y.max():\n            y = y - shift_space\n        else:\n            y = y + shift_space\n        return([x, y])    \n\n    \n    \n    def __generate_sin_and_noise(self, imgs):\n        \"\"\"\n         returns -> [h1_img, l1_img]\n        \"\"\"\n        x, y = self.__generate_sin(x_max = imgs[0].shape[1])\n        coords = np.column_stack((x, y))\n        \n        return_imgs = []\n        for i, img in enumerate(imgs):\n            img_noise = self.__gen_noise_by_shuffle_each_column(img)\n            noise_sig =round(random.uniform(CFG[\"SIGNAL_LVL\"][0], CFG[\"SIGNAL_LVL\"][1]), 10)\n            if i == 0:\n                self.s1 = noise_sig\n            else:\n                self.s2 = noise_sig\n            img = np.zeros(img_noise.shape)\n            sin_img = cv2.polylines(img, np.int32([coords]), False, color=255, thickness=1, lineType=cv2.LINE_AA)/255.\n            img = sin_img*noise_sig + img_noise\n            return_imgs.append(img)\n        return(return_imgs)\n    \n    \n    def __norm_cpu(self, x):\n        return (x - x.min())/((x.max() - x.min()))\n\n    \n    def __getitem__(self, idx):\n        sample = self.__samples[idx]\n        h1_img= sample.h1\n        l1_img= sample.l1\n\n        if self.train:\n            if random.randrange(0, 100) < 50:\n                label = 0\n                h1_img = self.__gen_noise_by_shuffle_each_column(h1_img)\n                h1_img = np.expand_dims(h1_img, axis=0)\n                l1_img = self.__gen_noise_by_shuffle_each_column(l1_img)\n                l1_img = np.expand_dims(l1_img, axis=0)\n            else:\n                label = 1\n                label = self.s1 + self.s2\n                h1_img, l1_img = self.__generate_sin_and_noise([h1_img, l1_img])\n        else:\n            label = sample.label\n            \n#         h1_img = self.__norm_cpu(h1_img)\n#         l1_img = self.__norm_cpu(h1_img)\n#         h1l1_img = np.stack((h1_img,l1_img), axis=1).squeeze()\n        img = self.__preprocess(h1_img, l1_img).squeeze()\n        return img, label, idx\n","metadata":{"executionInfo":{"elapsed":259,"status":"ok","timestamp":1669050970862,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"7y4lABUVZrzY","execution":{"iopub.status.busy":"2023-06-14T17:14:38.062008Z","iopub.execute_input":"2023-06-14T17:14:38.063217Z","iopub.status.idle":"2023-06-14T17:14:38.118061Z","shell.execute_reply.started":"2023-06-14T17:14:38.063179Z","shell.execute_reply":"2023-06-14T17:14:38.117046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = G2Dataset(CFG['DATASET'], preload=False, segmentation_mode=False)\nlen(dataset)","metadata":{"executionInfo":{"elapsed":7642,"status":"ok","timestamp":1669050980696,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"B9O4bJm4Zr3n","outputId":"a5546145-4a8b-4666-d983-724deb882342","execution":{"iopub.status.busy":"2023-06-14T17:14:38.121522Z","iopub.execute_input":"2023-06-14T17:14:38.121813Z","iopub.status.idle":"2023-06-14T17:15:52.213874Z","shell.execute_reply.started":"2023-06-14T17:14:38.121788Z","shell.execute_reply":"2023-06-14T17:15:52.212833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%matplotlib inline\n\n\n@interact\ndef show(s=(0, len(dataset)-1)):\n    dataset.train=True\n    try:\n        img, label, idx = dataset[s]\n        print(\"Label:\", label)\n    except:\n        img, idx = dataset[s]\n        \n    print(\"Shape\", img.shape)\n#     img = img.transpose(1,2,0)\n    print(\"i:\", s)\n    print(img.min(), img.max(), img.std())\n    \n    plt.figure(figsize=(10,10))\n    plt.imshow(img[0])\n    plt.figure(figsize=(10,10))\n    \n    plt.imshow(img[1])\n    #536","metadata":{"executionInfo":{"elapsed":587,"status":"ok","timestamp":1669047432396,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"r9aEdSwCZr6j","outputId":"9b865ae7-812e-4aa0-a7ed-7cf62ff865f6","execution":{"iopub.status.busy":"2023-06-14T17:15:52.215497Z","iopub.execute_input":"2023-06-14T17:15:52.216544Z","iopub.status.idle":"2023-06-14T17:15:57.373236Z","shell.execute_reply.started":"2023-06-14T17:15:52.216503Z","shell.execute_reply":"2023-06-14T17:15:57.372328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split data to train and validation","metadata":{"id":"kMIE_iZQaIsn"}},{"cell_type":"markdown","source":"### New split by validation folder","metadata":{}},{"cell_type":"code","source":"valid_labels = []\ntrain_labels = []\n\nfor i in range(len(dataset)):\n    valid = dataset.at(i).valid\n    valid_labels.append(i) if valid else train_labels.append(i)\n        \ndata_trn = Subset(dataset, train_labels)\ndata_val = Subset(dataset, valid_labels)\n\nloader_trn = DataLoader(data_trn, batch_size=CFG[\"BATCH_SIZE\"], shuffle=True,  pin_memory=True)    # num_workers=CFG[\"num_workers\"]\nloader_val = DataLoader(data_val, batch_size=CFG[\"BATCH_SIZE\"], shuffle=False, pin_memory=True)\n        \nprint(f'Train set: {len(data_trn)}')\n#dataset_stat(data_trn)\n\n\nprint(f'Valid set: {len(data_val)}')\n#dataset_stat(data_val)","metadata":{"execution":{"iopub.status.busy":"2022-12-29T09:11:59.827074Z","iopub.execute_input":"2022-12-29T09:11:59.829902Z","iopub.status.idle":"2022-12-29T09:11:59.850387Z","shell.execute_reply.started":"2022-12-29T09:11:59.82986Z","shell.execute_reply":"2022-12-29T09:11:59.849286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{"id":"1sayihu8aIgB"}},{"cell_type":"markdown","source":"## EffNet B0","metadata":{}},{"cell_type":"code","source":"torch.cuda.empty_cache()\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ndevice","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1669047650361,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"aQxaUAuuaSXm","outputId":"556ca9ff-668c-42a4-ef1f-d806fa40bf7e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EfficientNet_B0(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        #self.conv = nn.Conv2d(1, 3, kernel_size=(1, 1), padding='same')        \n        effnet = torchvision.models.efficientnet_b0(pretrained=True).to(device)\n        self.model = create_feature_extractor(effnet, ['flatten'])\n        freeze = False\n        if freeze:\n            for param in self.model.parameters():\n                param.requires_grad = False\n        self.linear = torch.nn.Linear(1280, 1)\n\n    def forward(self, x):\n        #out = self.conv(x)                                  \n#         out = torch.cat([x,x,x],dim=1)\n#         out = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])(out)\n#         print(out.min(), out.max(), out.std())\n        out = self.model(x)['flatten']\n        out = self.linear(out)\n        return out\n\n    def predict(self, x):\n        out = self.forward(x)\n        out = torch.sigmoid(out)\n        return out","metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1669050911799,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"2B4xzxFdaYOk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG['LR']           = 1e-4            # learning rate\nCFG['L2']           = 0               # L2 regularization 1e-3\nCFG['NUM_STEPS']    = 500             # steps per epoch\nCFG['DROPOUT']      = 0.2             # dropout\nCFG['WEIGHTS_PATH'] = 'C:/Users/semen/Desktop/job_folder/G2Net/Colab_stuff/model_weights/_07_12_2022___epochs_300___signal_lvl_0.03-0.15__L1_light.pth'\n# CFG['WEIGHTS']    = torchvision.EfficientNet_B0_Weights.DEFAULT  # weights None 'IMAGENET1K_V1' 'IMAGENET1K_V2'\n\n\nmodel = EfficientNet_B0()\nmodel.to(device)\n\n# model.load_state_dict(torch.load(CFG['WEIGHTS_PATH'], map_location=torch.device('cpu')))\nmodel.load_state_dict(torch.load(CFG['WEIGHTS_PATH']))\n\n#AUC\nauc_fn = BinaryAUROC()\n\n# ADAM Optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=CFG['LR'], weight_decay=CFG['L2'])   \n\ntry:\n    img, label, idx = dataset[0]\nexcept:\n    img, idx = dataset[0]\n# Model statistics\nsummary(model, input_size=(CFG[\"BATCH_SIZE\"], 3, img.shape[1], img.shape[2]), col_names=[\"input_size\", \"output_size\",\"num_params\"])","metadata":{"executionInfo":{"elapsed":416,"status":"ok","timestamp":1669050912779,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"PMk21nNoaaa5","outputId":"9789472f-606d-4c5c-f385-e976c7ba9bc3"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EffNet B1","metadata":{}},{"cell_type":"code","source":"class EfficientNet_B1(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        #self.conv = nn.Conv2d(1, 3, kernel_size=(1, 1), padding='same')        \n        effnet = torchvision.models.efficientnet_b1(pretrained=True).to(device)\n        self.model = create_feature_extractor(effnet, ['flatten'])\n        freeze = False\n        if freeze:\n            for param in self.model.parameters():\n                param.requires_grad = False\n        self.linear = torch.nn.Linear(1280, 1)\n\n    def forward(self, x):\n        #out = self.conv(x)                                  \n#         out = torch.cat([x,x,x],dim=1)\n#         out = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])(out)\n#         print(out.min(), out.max(), out.std())\n        out = self.model(x)['flatten']\n        out = self.linear(out)\n        return out\n\n    def predict(self, x):\n        out = self.forward(x)\n        out = torch.sigmoid(out)\n        return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG['LR']           = 1e-4            # learning rate\nCFG['L2']           = 0               # L2 regularization 1e-3\nCFG['NUM_STEPS']    = 500             # steps per epoch\nCFG['DROPOUT']      = 0.2             # dropout\nCFG['WEIGHTS_PATH'] = 'C:/Users/semen/Desktop/job_folder/G2Net/Colab_stuff/model_weights/epochs_500_noise_0.7_EffNet_b0_BCE_lr-4.pth'\n# CFG['WEIGHTS']    = torchvision.EfficientNet_B0_Weights.DEFAULT  # weights None 'IMAGENET1K_V1' 'IMAGENET1K_V2'\n\n\nmodel = EfficientNet_B1()\nmodel.to(device)\n#AUC\nauc_fn = BinaryAUROC()\n\n# ADAM Optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=CFG['LR'], weight_decay=CFG['L2'])   \n\n# Model statistics\nsummary(model, input_size=(CFG[\"BATCH_SIZE\"], 3, CFG[\"INPUT_SHAPE\"][1], CFG[\"INPUT_SHAPE\"][2]), col_names=[\"input_size\", \"output_size\",\"num_params\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Efficientnet V2","metadata":{"id":"MjCDo7CI7pfI"}},{"cell_type":"code","source":"CFG['LR']      = 0.0005 # 0.0005\nCFG['L2']      = 0\nCFG['DROPOUT'] = 0.20\n\nclass Effnetv2Model(torch.nn.Module):\n    def __init__(self, efnet_weights):\n        super().__init__()\n        effnet = torchvision.models.efficientnet_v2_s(weights=efnet_weights)\n        self.model  = create_feature_extractor(effnet, ['flatten'])\n        self.conv   = nn.Conv2d(2,3, kernel_size=3)\n        self.linear = torch.nn.Linear(1280, CFG[\"SAMPLE_FEATURES\"] )\n        self.act = torch.nn.ReLU()\n        #self.act = torch.nn.Tanh()\n        \n    def forward(self, x):\n        out = self.conv(x)\n        out = self.model(out)['flatten']\n        out = self.linear(out)\n        #out = self.act(out)\n        return out\n\n\nmodel = Effnetv2Model(torchvision.models.EfficientNet_V2_S_Weights.DEFAULT)\n#model = Effnetv2Model(None)\nmodel.to(device)\n\n# ADAM Optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=CFG['LR'], weight_decay=0) \n\n# Loss function\nloss_fn = nn.MSELoss()\n\n#AUC\nauc_fn = BinaryAUROC()\n\n# Model statistics\nsummary(model, input_size=(CFG[\"BATCH_SIZE\"], CFG[\"INPUT_SHAPE\"][0], CFG[\"INPUT_SHAPE\"][1], CFG[\"INPUT_SHAPE\"][1]), col_names=[\"output_size\",\"kernel_size\",\"num_params\"])\n\n    \n","metadata":{"id":"iJvqxkC77qjp","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"id":"7S89bb2oaIdR"}},{"cell_type":"code","source":"%matplotlib inline\ndef plot_loss_curves(results):\n    # Get the loss values of the results dictionary (training and test)\n    \n#     loss = [results['train_loss'][i].item() for i, _ in enumerate(results['train_loss'])]\n    test_loss = [results['test_loss'][i].item() for i, _ in enumerate(results['test_loss'])]                         # --------- Uncomment\n\n#     auc_metric_train = [results['auc_metric_train'][i].item() for i, _ in enumerate(results['auc_metric_train'])]\n    auc_metric_test = [results['auc_metric_test'][i].item() for i, _ in enumerate(results['auc_metric_test'])]       # --------- Uncomment\n\n\n    # Get the accuracy values of the results dictionary (training and test)\n#     accuracy = results['train_acc']\n    test_accuracy = results['test_acc']                          # --------- Uncomment\n\n\n    # Figure out how many epochs there were\n    epochs = range(len(results['train_loss']))\n\n    # Setup a plot \n    plt.figure(figsize=(25, 5), facecolor ='w')\n        \n    # Plot loss\n    plt.subplot(1, 3, 1)\n    plt.ylim(0, 1.00)\n    plt.xlim(0, CFG[\"NUM_EPOCHS\"])\n#     plt.plot(epochs, loss, label='train_loss')\n    plt.plot(epochs, test_loss, label='test_loss')              # --------- Uncomment\n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.legend()\n    plt.grid(color='gray', linestyle='--', alpha=0.6)\n    \n    # Plot accuracy\n    plt.subplot(1, 3, 2)\n    plt.ylim(0, 1.00)\n    plt.xlim(0, CFG[\"NUM_EPOCHS\"])\n#     plt.plot(epochs, accuracy, label='train_accuracy')\n    plt.plot(epochs, test_accuracy, label='test_accuracy')       # --------- Uncomment\n    plt.title('Accuracy')\n    plt.xlabel('Epochs')\n    plt.legend();\n    plt.grid(color='gray', linestyle='--', alpha=0.6)\n    \n    # Plot AUC\n    plt.subplot(1, 3, 3)\n    plt.ylim(0, 1.00)\n    plt.xlim(0, CFG[\"NUM_EPOCHS\"])\n#     plt.plot(epochs, auc_metric_train, label='AUC_train')\n    plt.plot(epochs, auc_metric_test, label='AUC_test')           # --------- Uncomment\n    plt.title('AUC')\n    plt.xlabel('Epochs')\n    plt.legend();\n    plt.grid(color='gray', linestyle='--', alpha=0.6)\n    \n    plt.show()\n# plot_loss_curves(results)","metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1669050947037,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"GHFf8GVXaewx","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_model_weights(model, name):\n    # 1. Create models directory \n    MODEL_PATH = Path(CFG['MODEL_WEIGHTS_PATH'])\n    MODEL_PATH.mkdir(parents=True, exist_ok=True)\n\n    # 2. Create model save path \n    MODEL_NAME = f\"{name}.pth\"\n    MODEL_SAVE_PATH = MODEL_PATH / MODEL_NAME\n\n    # 3. Save the model state dict \n    print(f\"Saving model to: {MODEL_SAVE_PATH}\")\n    torch.save(obj=model.state_dict(), # only saving the state_dict() only saves the models learned parameters\n               f=MODEL_SAVE_PATH)","metadata":{"executionInfo":{"elapsed":1,"status":"ok","timestamp":1669050948046,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"qAZyi4rjaglK","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step(model: torch.nn.Module, \n               dataloader: torch.utils.data.DataLoader, \n               loss_fn: torch.nn.Module, \n               optimizer: torch.optim.Optimizer,\n               device: torch.device):\n    \n    # Put model in train mode\n    model.train()\n    dataset.train=True\n\n    # Setup train loss and train accuracy values\n    train_loss, train_acc, auc_metric_train = 0, 0, 0\n    \n    s = 0\n    # Loop through data loader data batches\n    for batch, (X, y, idx) in enumerate(dataloader):\n\n        X = X.float()\n        y = y.float()\n        X, y = X.to(device), y.to(device)\n\n        # 1. Forward pass\n        train_pred_logits = model(X).squeeze()\n        \n        # 2. Calculate  and accumulate loss\n#         loss = nn.functional.binary_cross_entropy_with_logits(train_pred_logits, y)\n        loss = loss_fn(train_pred_logits, y)\n        \n        # 3. Optimizer zero grad\n        optimizer.zero_grad()\n        # 4. Loss backward\n        loss.backward()\n        # 5. Optimizer step\n        optimizer.step()\n\n        train_loss += loss.detach()\n        \n        y_auc= [0. if i == 0  else 1. for i in y]\n        y_auc = torch.tensor(y_auc).to(device)\n        \n        # Calculate and accumulate accuracy metric across all batches\n        y_pred_class = torch.sigmoid(train_pred_logits)\n        y_pred_class= [0. if i <=0.5 else 1. for i in y_pred_class]\n        train_acc += (torch.Tensor(y_pred_class).to(device) == y_auc).float().sum()/len(train_pred_logits)\n        # CalculateAUC\n        auc_train = auc_fn(train_pred_logits, y_auc.to(device))\n        auc_metric_train += auc_train  \n         \n        y_auc = None\n        \n    # Adjust metrics to get average loss and accuracy per batch \n    train_loss = train_loss.cpu() / len(dataloader)\n    train_acc = train_acc.cpu() / len(dataloader)\n    auc_metric_train = auc_metric_train.cpu() / len(dataloader)\n    return train_loss, train_acc, auc_metric_train\n\n\ndef test_step(model: torch.nn.Module, \n              dataloader: torch.utils.data.DataLoader, \n              loss_fn: torch.nn.Module,\n              device: torch.device):\n    # Put model in eval mode\n    model.eval() \n    dataset.train=False\n    # Setup test loss and test accuracy values\n    test_loss, test_acc, auc_metric_test = 0, 0, 0\n            \n    # Turn on inference context manager\n    with torch.inference_mode():\n        # Loop through DataLoader batches\n        for batch, (X, y, idx) in enumerate(dataloader):\n            # Send data to target device\n            X = X.float()\n            y = y.float()\n            X, y = X.to(device), y.to(device)\n\n            # 1. Forward pass\n            test_pred_logits = model(X).squeeze()\n\n            # 2. Calculate and accumulate loss\n#             loss = nn.functional.binary_cross_entropy_with_logits(test_pred_logits, y)\n            loss = loss_fn(test_pred_logits, y)\n            test_loss += loss\n\n            # Calculate and accumulate accuracy\n            y_pred_class = torch.sigmoid(test_pred_logits)\n            y_pred_class= [0. if i <=0.5 else 1. for i in y_pred_class]\n            test_acc += (torch.Tensor(y_pred_class).to(device) == y).float().sum()/len(test_pred_logits)\n            \n            # CalculateAUC\n            auc_test = auc_fn(test_pred_logits, y)\n            auc_metric_test += auc_test\n    \n    # Adjust metrics to get average loss and accuracy per batch \n    test_loss = test_loss.cpu() / len(dataloader)\n    test_acc = test_acc.cpu() / len(dataloader)\n    auc_metric_test = auc_metric_test.cpu() / len(dataloader)\n    return test_loss, test_acc, auc_metric_test\n\n\ndef train(model: torch.nn.Module, \n            train_dataloader: torch.utils.data.DataLoader, \n            test_dataloader: torch.utils.data.DataLoader, \n            optimizer: torch.optim.Optimizer,\n            loss_fn: torch.nn.Module,\n            epochs: int,\n            device: torch.device):\n\n    # Create empty results dictionary\n    results = {\"train_loss\":      [],\n               \"train_acc\":       [],\n               \"test_loss\":       [],\n               \"test_acc\":        [],\n               \"auc_metric_train\":[],\n               \"auc_metric_test\": []\n    }\n    \n    best_AUC  = 0\n    \n    for epoch in tqdm(range(epochs)):\n        if False:\n        train_loss, train_acc, auc_metric_train = train_step(model=model,\n                                                          dataloader=train_dataloader,\n                                                          loss_fn=loss_fn,\n                                                          optimizer=optimizer,\n                                                          device=device)\n\n        test_loss, test_acc, auc_metric_test = test_step(model=model,\n                                                           dataloader=test_dataloader,\n                                                           loss_fn=loss_fn,\n                                                           device=device)\n\n        \n        \n        # Update results dictionary\n        results[\"train_loss\"].append(train_loss)\n        results[\"train_acc\"].append(train_acc)\n        results[\"test_loss\"].append(test_loss)\n        results[\"test_acc\"].append(test_acc)\n        results[\"auc_metric_train\"].append(auc_metric_train)\n        results[\"auc_metric_test\"].append(auc_metric_test)\n        \n        # Print out what's happening\n        if auc_metric_test >= best_AUC:\n            best_AUC = auc_metric_test\n            best_epoch = epoch+1\n            save_model_weights(model=model, name=CFG['WEIGHTS_NAME'])\n        else:\n            save_model_weights(model=model, name=CFG['WEIGHTS_NAME_BACKUP'])\n         \n        \n        \n        \n        print(f\"Epoch: {epoch+1} |  ---------- BEST_AUC: {best_AUC:.4f} on epoch: {best_epoch} \\n\"\n              f\"TRAIN: loss: {train_loss:.4f} | \"\n              f\"acc: {train_acc:.4f} | \"\n              f\"AUC: {auc_metric_train:.4f} \\n\"\n              f\"TEST: loss: {test_loss:.4f} | \"\n              f\"acc: {test_acc:.4f}| \"\n              f\"AUC: {auc_metric_test:.4f} \")\n        \n\n\n        if epoch%3==0:\n            plot_loss_curves(results)\n        if epoch+1==CFG[\"NUM_EPOCHS\"]:\n            plot_loss_curves(results)\n            \n            \n    # Return the filled results at the end of the epochs\n    return results","metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1669050990744,"user":{"displayName":"Semen Bondarenko","userId":"09539615829692559589"},"user_tz":-120},"id":"qPpyZRawaiA_","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Train model\nresults = train(model=model,\n                train_dataloader=loader_trn,\n                test_dataloader=loader_val,\n                optimizer=optimizer,\n                loss_fn=loss_fn,\n                epochs=CFG[\"NUM_EPOCHS\"],\n                device=device)\n\n","metadata":{"id":"q35quxPrakDv","outputId":"bf0ca979-83a2-42c8-e7b5-998a7e50fb9c","scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate and create submit file","metadata":{}},{"cell_type":"code","source":"def evaluate_sample(X, predicted_label): \n    model.eval()\n    dataset.train=False\n    with torch.inference_mode():\n        X = torch.from_numpy(X).float()\n        X = X.unsqueeze(dim=0)\n        X = X.to(device)\n        test_pred_logits = model(X)\n        test_pred_labels = test_pred_logits.sigmoid().squeeze()\n        predicted_label.append(test_pred_labels.cpu().detach().numpy().item())\n        \n        \npredicted_label = [] \n\ndataset = G2Dataset(CFG['DATASET'], preload=True)\nfor i in tqdm(range(len(dataset))):\n    X = dataset[i][0]\n    evaluate_sample(X=X, predicted_label=predicted_label)","metadata":{"id":"iCqDaEpBal5r","outputId":"f73f6787-353a-4178-c777-4a71e73aa1b2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_label = [] \n\ndataset = G2Dataset(CFG['DATASET'], preload=True)\ndata_val = Subset(dataset, range(len(dataset)))\nloader_val = DataLoader(data_val, batch_size=CFG[\"BATCH_SIZE\"], shuffle=False, pin_memory=True)\n\n\n# Put model in eval mode\nmodel.eval() \ndataset.train=False\n\nwith torch.inference_mode():\n    for batch, (X, idx) in enumerate(dataloader):\n        X = X.float()\n        X = X.to(device)\n        # 1. Forward pass\n        test_pred_logits = model(X)\n        test_pred_labels = test_pred_logits.sigmoid().squeeze()\n        predicted_label.append(test_pred_labels.cpu().detach().numpy().item())\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"tp-V6vstaA34"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"C9JwPbBPaA1r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"xAzGukOWaAyv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"k04B3bd2aAuX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"lTYvCIcZaArk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"oWR7atVCaAnB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"SccfnnAhaAka"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"Oskzqrp6aAht"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"-30_6nIraAan"},"execution_count":null,"outputs":[]}]}