{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# setting kernel type\n\nto use tpu set training = 'tpu' for gpu training = 'gpu' for cpu training = 'cpu'\n\n# **Note for TPU\n**while these notebook is ok to work in tpu , the xla for pytorch As per they have announced, its still under development and so for operations not supported they automatically switch to CPU so it might get even slower than GPU.\n\nso for some reson in these notebook when using this data sets where we have large size of images when image resize and transformations it will use cpu for that transformation and do training in tpu and these will cost memory to work around with these either you save images after transformation and read them diractly into the model or i just skip val phase and also make batches very small in tpu \nto use tpu in these dataset\njust set profile='low' other can set profile=''high'","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training = 'gpu'\nprofile = 'high' # for this dtata set to use tpu set these var to low","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:19:45.044529Z","iopub.execute_input":"2022-08-13T06:19:45.044904Z","iopub.status.idle":"2022-08-13T06:19:45.143735Z","shell.execute_reply.started":"2022-08-13T06:19:45.044824Z","shell.execute_reply":"2022-08-13T06:19:45.142984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# install lighting and xla for tpu","metadata":{}},{"cell_type":"code","source":"if training =='tpu':\n !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n !python pytorch-xla-env-setup.py --version 1.7 --apt-packages libomp5 libopenblas-dev","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:19:52.503254Z","iopub.execute_input":"2022-08-13T06:19:52.503982Z","iopub.status.idle":"2022-08-13T06:19:52.5091Z","shell.execute_reply.started":"2022-08-13T06:19:52.503947Z","shell.execute_reply":"2022-08-13T06:19:52.508358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install inplace-abn","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:19:53.23186Z","iopub.execute_input":"2022-08-13T06:19:53.232739Z","iopub.status.idle":"2022-08-13T06:19:53.237701Z","shell.execute_reply.started":"2022-08-13T06:19:53.232695Z","shell.execute_reply":"2022-08-13T06:19:53.236165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install pytorch-lightning #==1.1.8\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-13T06:19:53.679953Z","iopub.execute_input":"2022-08-13T06:19:53.680717Z","iopub.status.idle":"2022-08-13T06:20:01.764569Z","shell.execute_reply.started":"2022-08-13T06:19:53.680679Z","shell.execute_reply":"2022-08-13T06:20:01.763583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:01.766784Z","iopub.execute_input":"2022-08-13T06:20:01.76704Z","iopub.status.idle":"2022-08-13T06:20:09.135034Z","shell.execute_reply.started":"2022-08-13T06:20:01.767009Z","shell.execute_reply":"2022-08-13T06:20:09.134156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# library needed by lightining","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.nn import functional as F\nfrom torch import nn\nfrom pytorch_lightning.core import LightningModule\nfrom sklearn.metrics import label_ranking_average_precision_score\n\nprint(torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:09.136725Z","iopub.execute_input":"2022-08-13T06:20:09.137021Z","iopub.status.idle":"2022-08-13T06:20:15.971781Z","shell.execute_reply.started":"2022-08-13T06:20:09.136972Z","shell.execute_reply":"2022-08-13T06:20:15.970483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# other libraarys","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nfrom torchvision import datasets, models, transforms\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torchvision import datasets, models, transforms \nfrom torch.utils.data.sampler import SubsetRandomSampler \nfrom torch.utils.data import Dataset, DataLoader\n\nimport pandas as pd\nimport torch.nn.functional as F\n\nimport time\nimport os\nimport time\nimport random\nfrom datetime import datetime\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\n\nimport timm\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint , EarlyStopping\nfrom skimage.feature import hog , blob_dog, blob_log, blob_doh , daisy , ORB\nfrom skimage import data, exposure\nfrom skimage.color import rgb2gray , rgba2rgb\nfrom math import sqrt\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:15.974265Z","iopub.execute_input":"2022-08-13T06:20:15.974555Z","iopub.status.idle":"2022-08-13T06:20:17.215648Z","shell.execute_reply.started":"2022-08-13T06:20:15.97452Z","shell.execute_reply":"2022-08-13T06:20:17.214818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# seeding everthing","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n \"\"\"\n Seeds basic parameters for reproductibility of results\n \n Arguments:\n seed {int} -- Number of the seed\n \"\"\"\n random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n if training == 'gpu':\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\nseed_everything(999)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.216923Z","iopub.execute_input":"2022-08-13T06:20:17.217189Z","iopub.status.idle":"2022-08-13T06:20:17.22624Z","shell.execute_reply.started":"2022-08-13T06:20:17.217153Z","shell.execute_reply":"2022-08-13T06:20:17.225574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# configartions","metadata":{}},{"cell_type":"code","source":"DATA_PATH = \"../input/mayo-clinic-strip-ai/\"\nTRAIN_PATH = \"../input/jpg-images-strip-ai/train/\" #\"../input/mayo-clinic-strip-ai/train/\"\nTEST_PATH = \"../input/mayo-clinic-strip-ai/test/\"\n\n\n\n\nIMG_SIZE = 384\nif profile =='low':\n BATCH_SIZE = 8 \n val_BATCH_SIZE = 8\n valchecking = 3\n epoch = 2\n monit = 'train_loss'\nelse:\n BATCH_SIZE = 10\n val_BATCH_SIZE = 10\n valchecking = 1\n epoch = 20\n monit = 'val_loss'","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.227654Z","iopub.execute_input":"2022-08-13T06:20:17.228036Z","iopub.status.idle":"2022-08-13T06:20:17.237769Z","shell.execute_reply.started":"2022-08-13T06:20:17.227998Z","shell.execute_reply":"2022-08-13T06:20:17.236952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# read csv file with pandas","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.239017Z","iopub.execute_input":"2022-08-13T06:20:17.239837Z","iopub.status.idle":"2022-08-13T06:20:17.276055Z","shell.execute_reply.started":"2022-08-13T06:20:17.239802Z","shell.execute_reply":"2022-08-13T06:20:17.275376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.277356Z","iopub.execute_input":"2022-08-13T06:20:17.27784Z","iopub.status.idle":"2022-08-13T06:20:17.293796Z","shell.execute_reply.started":"2022-08-13T06:20:17.277806Z","shell.execute_reply":"2022-08-13T06:20:17.292911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.295371Z","iopub.execute_input":"2022-08-13T06:20:17.295721Z","iopub.status.idle":"2022-08-13T06:20:17.308993Z","shell.execute_reply.started":"2022-08-13T06:20:17.295669Z","shell.execute_reply":"2022-08-13T06:20:17.308058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts().plot(kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.31186Z","iopub.execute_input":"2022-08-13T06:20:17.312107Z","iopub.status.idle":"2022-08-13T06:20:17.530702Z","shell.execute_reply.started":"2022-08-13T06:20:17.312081Z","shell.execute_reply":"2022-08-13T06:20:17.530005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['y'] = -1","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.532134Z","iopub.execute_input":"2022-08-13T06:20:17.532586Z","iopub.status.idle":"2022-08-13T06:20:17.537741Z","shell.execute_reply.started":"2022-08-13T06:20:17.532548Z","shell.execute_reply":"2022-08-13T06:20:17.536873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.y[df['label'] == 'CE'] = 0\ndf.y[df['label'] == 'LAA'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.538867Z","iopub.execute_input":"2022-08-13T06:20:17.539393Z","iopub.status.idle":"2022-08-13T06:20:17.554842Z","shell.execute_reply.started":"2022-08-13T06:20:17.539337Z","shell.execute_reply":"2022-08-13T06:20:17.553685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.y.min()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.557273Z","iopub.execute_input":"2022-08-13T06:20:17.558309Z","iopub.status.idle":"2022-08-13T06:20:17.565159Z","shell.execute_reply.started":"2022-08-13T06:20:17.558272Z","shell.execute_reply":"2022-08-13T06:20:17.564379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.y.max()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.566666Z","iopub.execute_input":"2022-08-13T06:20:17.566893Z","iopub.status.idle":"2022-08-13T06:20:17.575517Z","shell.execute_reply.started":"2022-08-13T06:20:17.566869Z","shell.execute_reply":"2022-08-13T06:20:17.574714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.576534Z","iopub.execute_input":"2022-08-13T06:20:17.576979Z","iopub.status.idle":"2022-08-13T06:20:17.596189Z","shell.execute_reply.started":"2022-08-13T06:20:17.576947Z","shell.execute_reply":"2022-08-13T06:20:17.595499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# split data","metadata":{}},{"cell_type":"code","source":"train_df, valid_df = model_selection.train_test_split(\n df, test_size=0.15, random_state=999 , stratify=df.y.values\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.597594Z","iopub.execute_input":"2022-08-13T06:20:17.597875Z","iopub.status.idle":"2022-08-13T06:20:17.609901Z","shell.execute_reply.started":"2022-08-13T06:20:17.597845Z","shell.execute_reply":"2022-08-13T06:20:17.60911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data set class for getting data from path","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#oversampling using wighted class\nimport numpy as np\nfrom torch.utils.data.sampler import WeightedRandomSampler\n\ncounts = np.bincount(train_df['y'])\nlabels_weights = 1. / counts\nweights = labels_weights[train_df['y']]\nsampler = WeightedRandomSampler(torch.DoubleTensor(weights), len(weights))\n","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.611203Z","iopub.execute_input":"2022-08-13T06:20:17.61169Z","iopub.status.idle":"2022-08-13T06:20:17.624413Z","shell.execute_reply.started":"2022-08-13T06:20:17.611657Z","shell.execute_reply":"2022-08-13T06:20:17.623447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n \nclass TaskDataset(torch.utils.data.Dataset):\n \"\"\"\n Helper Class to create the pytorch dataset\n \"\"\"\n\n def __init__(self, df, data_path=DATA_PATH, mode=\"train\", transforms=None):\n super().__init__()\n self.df_data = df\n self.data_path = data_path\n self.transforms = transforms\n self.mode = mode\n self.data_dir = TRAIN_PATH if mode == \"train\" else TEST_PATH\n\n def __len__(self):\n return len(self.df_data)\n\n def __getitem__(self, index):\n img_name = self.df_data.image_id.values[index]\n img_path = os.path.join(self.data_dir, img_name + \".jpg\" )\n img = Image.open(img_path).convert(\"RGB\").resize((IMG_SIZE, IMG_SIZE))\n #image2 = np.array(img) \n fd = hog(np.array(img) , orientations=8, pixels_per_cell=(4, 4),\n cells_per_block=(1, 1) )\n fd = torch.tensor(fd,dtype= torch.float32)\n image_gray = rgb2gray(np.array(img) )\n '''blobs_log = blob_log(image_gray, min_sigma=30 , max_sigma=30, num_sigma=100, threshold=.1)\n # Compute radii in the 3rd column.\n blobs_log[:, 2] = blobs_log[:, 2] * sqrt(2)'''\n '''blobs_dog = blob_dog(image_gray, max_sigma=30, threshold=.1)\n blobs_dog[:, 2] = blobs_dog[:, 2] * sqrt(2)\n blobs_doh = blob_doh(image_gray, max_sigma=30, threshold=.01)'''\n '''print(blobs_log.shape)\n blobs_log = blobs_log.reshape(4*3 )\n blobs_log = torch.tensor(blobs_log,dtype= torch.float16)'''\n '''blobs_dog = blobs_dog.reshape(352*3)\n blobs_dog = torch.tensor(blobs_dog,dtype= torch.float16)\n blobs_doh = blob_doh.reshape(382*3)\n blobs_doh = torch.tensor(blobs_doh,dtype= torch.float16)'''\n '''daisy_desc, descs_img = daisy(image_gray, step=180, radius=58, rings=2, histograms=6,\n orientations=8, visualize=True)\n daisy_desc = daisy_desc.reshape(3*3*104)\n daisy_desc = torch.tensor(daisy_desc,dtype= torch.float32)'''\n descriptor_extractor = ORB(downscale=1.2, n_scales=8, n_keypoints=20, fast_n=9, \n fast_threshold=0.08, harris_k=0.04)\n descriptor_extractor.detect_and_extract(image_gray)\n desc_orb = descriptor_extractor.descriptors\n outt = np.full((20, 256), False)\n outt[:desc_orb.shape[0]][:desc_orb.shape[1]] = desc_orb[:][:]\n outt = outt.reshape(20 * 256)\n outt = torch.tensor(outt,dtype= torch.float32)\n if self.mode == \"train\":\n label = {\"CE\" : 0, \"LAA\": 1}[self.df_data.iloc[index].label]\n label = torch.tensor(label,dtype= torch.float16) \n if self.transforms is not None:\n img = self.transforms(img)\n out = { 'image' : img , 'hog': fd ,'desc_orb': outt }# ,'daisy_desc': daisy_desc ,'desc_orb': outt }\n if self.mode == \"train\":\n return out , label\n del image_gray\n \n return out ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.625995Z","iopub.execute_input":"2022-08-13T06:20:17.626293Z","iopub.status.idle":"2022-08-13T06:20:17.640238Z","shell.execute_reply.started":"2022-08-13T06:20:17.626261Z","shell.execute_reply":"2022-08-13T06:20:17.639439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ** transforming data doing augmention **","metadata":{}},{"cell_type":"code","source":"# Imagenet means and stds\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n \ntransforms_train = transforms.Compose([\n transforms.Resize((IMG_SIZE, IMG_SIZE)),\n transforms.RandomRotation(45),\n transforms.RandomHorizontalFlip(),\n transforms.RandomVerticalFlip(),\n transforms.ToTensor(),\n transforms.Normalize(mean=mean, std=std),\n # transforms.Grayscale(num_output_channels=1)\n ]\n )\n\ntransforms_valid = transforms.Compose(\n [\n transforms.Resize((IMG_SIZE, IMG_SIZE)),\n transforms.ToTensor(),\n transforms.Normalize(mean=mean, std=std),\n #transforms.Grayscale(num_output_channels=1)\n ]\n ) \n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:17.643135Z","iopub.execute_input":"2022-08-13T06:20:17.643328Z","iopub.status.idle":"2022-08-13T06:20:17.654664Z","shell.execute_reply.started":"2022-08-13T06:20:17.643305Z","shell.execute_reply":"2022-08-13T06:20:17.653762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make datasets and data loaders","metadata":{}},{"cell_type":"code","source":"train_dataset = TaskDataset(train_df, transforms=transforms_train)\nvalid_dataset = TaskDataset(valid_df, transforms=transforms_valid)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:20.102976Z","iopub.execute_input":"2022-08-13T06:20:20.103619Z","iopub.status.idle":"2022-08-13T06:20:20.109506Z","shell.execute_reply.started":"2022-08-13T06:20:20.103584Z","shell.execute_reply":"2022-08-13T06:20:20.108584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''del df\ndel train_df\ndel valid_df'''","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:20.862962Z","iopub.execute_input":"2022-08-13T06:20:20.863554Z","iopub.status.idle":"2022-08-13T06:20:20.86951Z","shell.execute_reply.started":"2022-08-13T06:20:20.863518Z","shell.execute_reply":"2022-08-13T06:20:20.868713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader( dataset=train_dataset,\n batch_size=BATCH_SIZE, \n \n drop_last=True,\n shuffle=False ,sampler = sampler)\nvalid_loader = DataLoader( dataset=valid_dataset,\n batch_size=val_BATCH_SIZE ,\n \n drop_last=True,\n shuffle=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:21.513023Z","iopub.execute_input":"2022-08-13T06:20:21.513299Z","iopub.status.idle":"2022-08-13T06:20:21.518176Z","shell.execute_reply.started":"2022-08-13T06:20:21.513269Z","shell.execute_reply":"2022-08-13T06:20:21.517469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs, classes = next(iter(train_loader)) \nprint(classes)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:22.329965Z","iopub.execute_input":"2022-08-13T06:20:22.330593Z","iopub.status.idle":"2022-08-13T06:20:31.300829Z","shell.execute_reply.started":"2022-08-13T06:20:22.330549Z","shell.execute_reply":"2022-08-13T06:20:31.299316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs['image'].shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:31.302436Z","iopub.execute_input":"2022-08-13T06:20:31.302859Z","iopub.status.idle":"2022-08-13T06:20:31.309096Z","shell.execute_reply.started":"2022-08-13T06:20:31.30282Z","shell.execute_reply":"2022-08-13T06:20:31.308357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# in these code we defin pytorchlightning model\n\n\n","metadata":{}},{"cell_type":"code","source":"gpu = torch.cuda.is_available()\ndevice = torch.device(\"cuda\" if gpu else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:35.782025Z","iopub.execute_input":"2022-08-13T06:20:35.782843Z","iopub.status.idle":"2022-08-13T06:20:35.850515Z","shell.execute_reply.started":"2022-08-13T06:20:35.782806Z","shell.execute_reply":"2022-08-13T06:20:35.849615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_lightning.loggers import TensorBoardLogger\n","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:38.2101Z","iopub.execute_input":"2022-08-13T06:20:38.210668Z","iopub.status.idle":"2022-08-13T06:20:38.214824Z","shell.execute_reply.started":"2022-08-13T06:20:38.210622Z","shell.execute_reply":"2022-08-13T06:20:38.213964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"typebackpone = 'layers'\nmodel_name = \"swin_base_patch4_window12_384\"\ninfet = 2048","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:41.352852Z","iopub.execute_input":"2022-08-13T06:20:41.35348Z","iopub.status.idle":"2022-08-13T06:20:41.357105Z","shell.execute_reply.started":"2022-08-13T06:20:41.353442Z","shell.execute_reply":"2022-08-13T06:20:41.356352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Mamonmodel(pl.LightningModule):\n def __init__(self):\n super().__init__()\n i = 0\n self.lr=1e-3\n self.wd = 1e-6\n self.schd = 'ReduceLROnPlateau'\n #self.best_model_wts = copy.deepcopy(model.state_dict())\n self.best_loss = np.Inf\n '''basemodel = timm.create_model(model_name, pretrained=True) #torch.hub.load('facebookresearch/deit:main', 'deit_base_patch16_224', pretrained=True) \n if typebackpone == 'layers':\n for param in basemodel.layer1.parameters():\n param.requires_grad = False\n for param in basemodel.layer2.parameters():\n param.requires_grad = False\n for param in basemodel.layer3.parameters():\n param.requires_grad = False\n for param in basemodel.layer4.parameters():\n param.requires_grad = True\n else: \n for child in basemodel.blocks.children():\n if i < 7:\n for param in child.parameters():\n param.requires_grad = False\n else:\n for param in child.parameters():\n param.requires_grad = True\n i +=1\n fcc = nn.Sequential(\n nn.Dropout(0.4),\n nn.Linear(infet, 1024),\n nn.Dropout(0.25),nn.ReLU(),\n \n )\n if typebackpone == 'layers':\n basemodel.fc = fcc\n else:\n basemodel.head = fcc\n '''\n '''basemodel = torch.hub.load('facebookresearch/deit:main', 'deit_base_patch16_384', pretrained=True)\n for child in basemodel.blocks.children():\n if i < 8:\n for param in child.parameters():\n param.requires_grad = False\n else:\n for param in child.parameters():\n param.requires_grad = True\n i +=1\n print(i)\n basemodel.head = nn.Sequential( \n \n nn.Linear(768, 256) ,\n nn.Dropout(0.2),nn.ReLU(),\n )\n self.model = basemodel'''\n self.model = timm.create_model(model_name, pretrained=True,\n in_chans=3)\n print(sum(p.numel() for p in self.model.parameters() if p.requires_grad))\n i = 1 \n for child in self.model.layers.children():\n #print(i)\n if i <= 3:\n for param in child.parameters():\n param.requires_grad = False\n else:\n #print('tranable')\n #print(i)\n for param in child.parameters():\n param.requires_grad = True\n i +=1\n print(i)\n n_features = self.model.head.in_features\n self.model.head = nn.Linear(n_features, 2133, bias=True)\n self.hog = nn.Linear(73728 , 1024 )\n self.daisy = nn.Linear(3*3*104 , 64 )\n self.orb = nn.Linear(5120 ,512)\n self.dropout1 = nn.Dropout(0.2)\n self.dropout2 = nn.Dropout(0.4)\n self.nn1 = nn.Linear(512 + 2133 +1024 , 1024)\n self.out = nn.Linear(1024 , 2)\n print(sum(p.numel() for p in self.parameters() if p.requires_grad))\n def forward(self,x ):\n hog = x['hog']\n '''daisy_desc = x['daisy_desc']'''\n desc_orb = x['desc_orb']\n x = x['image']\n x = F.relu(self.dropout2(self.model(x)))\n #featurs = torch.cat([extrametaa, featurs], dim=1)\n hog = F.relu(self.hog(hog))\n '''daisy_desc = F.relu(self.dropout1(self.daisy(daisy_desc)) )'''\n desc_orb = F.relu(self.orb(desc_orb))\n x = torch.cat([x, hog ,desc_orb ], dim=1)\n ''' x = torch.cat([x, daisy_desc], dim=1)\n x = torch.cat([x, desc_orb], dim=1)'''\n x = F.relu(self.nn1(x))\n x = self.dropout2(x)\n x = self.out(x)\n return x\n def training_step(self,batch,batch_idx):\n x , y = batch\n y = y.type(torch.LongTensor) # casting to long\n y = y.to(device)\n y_hat = self(x)\n loss = F.cross_entropy(y_hat, y) # F.binary_cross_entropy_with_logits(y_hat , y)\n #scor = label_ranking_average_precision_score(y.cpu(), torch.sigmoid(y_hat).cpu())\n self.log('train_loss', loss)\n #self.log('train_score', scor)\n return loss \n def validation_step(self,batch,batch_idx):\n x , y = batch\n y = y.type(torch.LongTensor) # casting to long\n y = y.to(device)\n y_hat = self(x)\n loss = F.cross_entropy(y_hat, y) # F.binary_cross_entropy_with_logits(y_hat , y)\n #scor = label_ranking_average_precision_score(y.cpu(), torch.sigmoid(y_hat).cpu())\n #self.log('val_score', scor)\n self.log('val_loss', loss) \n if self.best_loss > loss:\n self.best_loss = loss\n torch.save(self.state_dict(), 'swin384-bestwigh-ep-'+str(self.current_epoch)+'.pth')\n torch.save(self, 'swin384-best-model.pth')\n return {'val_loss': loss}\n def training_epoch_end(self, outputs):\n #print(outputs)\n avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n self.logger.experiment.add_scalar('train_loss',avg_loss, self.current_epoch)\n tr_avg.append(avg_loss)\n def validation_epoch_end(self, outputs):\n avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n self.logger.experiment.add_scalar('loss',avg_loss, self.current_epoch)\n avg.append(avg_loss)\n return {'av_loss' : avg_loss}\n def configure_optimizers(self):\n optimizer = optim.AdamW(\n self.parameters(), lr=self.lr, weight_decay=self.wd\n )\n if self.schd == 'ReduceLROnPlateau':\n optimizer = optim.Adam(\n self.parameters(), lr=self.lr \n )\n scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.7, patience=1,\n verbose=True)\n elif self.schd == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)\n elif self.schd == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=0.001, last_epoch=-1) \n\n return {'optimizer': optimizer, 'lr_scheduler': scheduler , 'monitor':monit }\n ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:42.325626Z","iopub.execute_input":"2022-08-13T06:20:42.326235Z","iopub.status.idle":"2022-08-13T06:20:42.350557Z","shell.execute_reply.started":"2022-08-13T06:20:42.326199Z","shell.execute_reply":"2022-08-13T06:20:42.349646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# init model\nmemomodel = Mamonmodel()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:20:44.5619Z","iopub.execute_input":"2022-08-13T06:20:44.562168Z","iopub.status.idle":"2022-08-13T06:21:03.898726Z","shell.execute_reply.started":"2022-08-13T06:20:44.562139Z","shell.execute_reply":"2022-08-13T06:21:03.898012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg = []\ntr_avg = []","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:21:07.202537Z","iopub.execute_input":"2022-08-13T06:21:07.20517Z","iopub.status.idle":"2022-08-13T06:21:07.210502Z","shell.execute_reply.started":"2022-08-13T06:21:07.205123Z","shell.execute_reply":"2022-08-13T06:21:07.209682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n{val_loss:.2f}\n'''\nearly_stop_callback = EarlyStopping(\n monitor='val_loss',\n min_delta=0.0000,\n patience=3,\n verbose=True,\n mode='min'\n )\nif training == 'gpu':\n checkpoint_callback = ModelCheckpoint(filename='{epoch}-gpumodel.pth', \n verbose=True,\n monitor='val_loss',\n mode='min')\n trainer = pl.Trainer(min_epochs=1 , max_epochs=epoch, check_val_every_n_epoch=valchecking,\n \n gpus=-1,callbacks=[checkpoint_callback , early_stop_callback]\n ) \n \nelif training == 'tpu':\n print('now we n tpu')\n checkpoint_callback = ModelCheckpoint(filename='{epoch}-tpumodel.pth', \n verbose=True,\n monitor=monit,\n mode='min')\n trainer = pl.Trainer(tpu_cores=8,min_epochs=1,num_sanity_val_steps=0 , check_val_every_n_epoch=valchecking , max_epochs=epoch, callbacks=[checkpoint_callback])\nelse:\n checkpoint_callback = ModelCheckpoint(filename='{epoch}-cpumodel.pth', \n verbose=True,\n monitor='val_loss',\n mode='min')\n trainer = pl.Trainer(min_epochs=1 , max_epochs=epoch,check_val_every_n_epoch=valchecking, callbacks=[checkpoint_callback , early_stop_callback])\n\n \n\n\ntrainer.fit(memomodel, train_loader , valid_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:21:07.550879Z","iopub.execute_input":"2022-08-13T06:21:07.551152Z","iopub.status.idle":"2022-08-13T06:23:54.451931Z","shell.execute_reply.started":"2022-08-13T06:21:07.551123Z","shell.execute_reply":"2022-08-13T06:23:54.451093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = {'val_avg' : [ x.item() for x in avg[1:] ] , 'train_avg ': [ xx.item() for xx in tr_avg ] } \nout_res = pd.DataFrame.from_dict(all_data)\nout_res.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T23:01:34.460171Z","iopub.status.idle":"2022-08-12T23:01:34.460583Z","shell.execute_reply.started":"2022-08-12T23:01:34.460359Z","shell.execute_reply":"2022-08-12T23:01:34.460378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}