{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Benchmark data-loading in Kernel\nBased on @jampia idea (https://www.kaggle.com/competitions/google-research-identify-contrails-reduce-global-warming/discussion/414549) I decided to benchmark loading code for different methods. The main difference here that I use float16 numpy array vs float32.\n\nThe speed is verified by running DataLoader twice and second run is taken into consideration.\n\nOption with scores:\n- raw numpy: 2:22\n- fastnumpy: 0:48\n- numpy-mem: 0:48\n- jpeg: 0:54\n\n\nThen Numpy-mem/fast seems like a best option as JPEG have quite similar speed. The only thing I haven't tested was CPU usage in these methods.","metadata":{}},{"cell_type":"code","source":"from typing import Any, Dict, Generator, List, Optional, Set, Union\nfrom pathlib import Path\n\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.auto import tqdm\nimport torchvision\n\nfrom torch.utils.data import Dataset, DataLoader\n\n\ntorch.__version__","metadata":{"execution":{"iopub.status.busy":"2023-06-10T07:04:37.682137Z","iopub.execute_input":"2023-06-10T07:04:37.682675Z","iopub.status.idle":"2023-06-10T07:04:38.01063Z","shell.execute_reply.started":"2023-06-10T07:04:37.682639Z","shell.execute_reply":"2023-06-10T07:04:38.009444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    train = True\n    num_epochs = 30\n    num_classes = 1\n    batch_size = 32\n    seed = 42\n    \n    encoder = 'efficientnet-b0'\n    pretrained = True\n    weights = 'imagenet'\n    classes = ['contrail']\n    activation = None\n    in_chans = 3\n    \n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    image_size = 256\n  \n    \nclass Paths:\n    data_root = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n    contrails = '/kaggle/input/contrails-images-ash-color/contrails/'\n    pngs = \"/kaggle/input/kaggle-contrails-png-09-06-23/contrails/\"\n    train_path = '/kaggle/input/contrails-images-ash-color/train_df.csv'\n    valid_path = '/kaggle/input/contrails-images-ash-color/valid_df.csv'","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:52:13.188604Z","iopub.execute_input":"2023-06-10T06:52:13.189151Z","iopub.status.idle":"2023-06-10T06:52:13.198888Z","shell.execute_reply.started":"2023-06-10T06:52:13.189122Z","shell.execute_reply":"2023-06-10T06:52:13.196774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import dataframes\ntrain_df = pd.read_csv(Paths.train_path)\nvalid_df = pd.read_csv(Paths.valid_path)\n\ntrain_df['path'] = Paths.contrails + train_df['record_id'].astype(str) + '.npy'\nvalid_df['path'] = Paths.contrails + valid_df['record_id'].astype(str) + '.npy'\n\ntrain_df.shape, valid_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:52:13.200618Z","iopub.execute_input":"2023-06-10T06:52:13.202747Z","iopub.status.idle":"2023-06-10T06:52:13.324181Z","shell.execute_reply.started":"2023-06-10T06:52:13.202712Z","shell.execute_reply":"2023-06-10T06:52:13.322891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class fastnumpyio:\n    def load(file:str) -> np.ndarray:\n        file=open(file,\"rb\")\n        header = file.read(128)\n        descr = str(header[19:25], 'utf-8').replace(\"'\",\"\").replace(\" \",\"\")\n        shape = tuple(int(num) for num in str(header[60:120], 'utf-8').replace(', }', '').replace('(', '').replace(')', '').split(','))\n        datasize = np.lib.format.descr_to_dtype(descr).itemsize\n        for dimension in shape:\n            datasize *= dimension\n        return np.ndarray(shape, dtype=descr, buffer=file.read(datasize))\n    \n\nclass memmap:\n    def load(file:str) -> np.ndarray:\n        return np.memmap(file, mode='r', dtype=np.float16, shape=(256,256,4))\n\n\nclass ContrailsDataset(torch.utils.data.Dataset):\n    def __init__(self, df, loader, train=True):\n        self.df = df\n        self.trn = train\n        self.loader = loader\n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        con_path = row.path\n        con = self.loader.load(str(con_path)).astype(float)\n        \n        img = con[..., :-1]\n        label = con[..., -1]\n        \n        img = torch.tensor(img)\n        label = torch.tensor(label)\n        \n        img = img.permute(2, 0, 1)\n        label = label.unsqueeze(0)        \n            \n        return img.float(), label.float()\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:52:13.327381Z","iopub.execute_input":"2023-06-10T06:52:13.328282Z","iopub.status.idle":"2023-06-10T06:52:13.341892Z","shell.execute_reply.started":"2023-06-10T06:52:13.32825Z","shell.execute_reply":"2023-06-10T06:52:13.34057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Numpy","metadata":{}},{"cell_type":"code","source":"train_ds = ContrailsDataset(\n        train_df,\n        train=True,\n        loader=np\n    )\n\ntrain_dl = DataLoader(train_ds, batch_size=Config.batch_size , shuffle=True, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:52:13.343683Z","iopub.execute_input":"2023-06-10T06:52:13.344123Z","iopub.status.idle":"2023-06-10T06:52:13.359756Z","shell.execute_reply.started":"2023-06-10T06:52:13.344086Z","shell.execute_reply":"2023-06-10T06:52:13.358301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(2):\n    for batch in tqdm(train_dl):\n        pass","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:52:13.361425Z","iopub.execute_input":"2023-06-10T06:52:13.362333Z","iopub.status.idle":"2023-06-10T06:57:50.91805Z","shell.execute_reply.started":"2023-06-10T06:52:13.362293Z","shell.execute_reply":"2023-06-10T06:57:50.915943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fast-numpy","metadata":{}},{"cell_type":"code","source":"train_ds = ContrailsDataset(\n        train_df,\n        train=True,\n        loader=fastnumpyio\n    )\n\ntrain_dl = DataLoader(train_ds, batch_size=Config.batch_size , shuffle=True, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:57:50.922036Z","iopub.execute_input":"2023-06-10T06:57:50.922653Z","iopub.status.idle":"2023-06-10T06:57:50.932314Z","shell.execute_reply.started":"2023-06-10T06:57:50.922591Z","shell.execute_reply":"2023-06-10T06:57:50.930865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(2):\n    for batch in tqdm(train_dl):\n        pass","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:57:50.933935Z","iopub.execute_input":"2023-06-10T06:57:50.934269Z","iopub.status.idle":"2023-06-10T06:59:33.039762Z","shell.execute_reply.started":"2023-06-10T06:57:50.934242Z","shell.execute_reply":"2023-06-10T06:59:33.03852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Numpy Memmap","metadata":{}},{"cell_type":"code","source":"train_ds = ContrailsDataset(\n        train_df,\n        train=True,\n        loader=memmap\n    )\n\ntrain_dl = DataLoader(train_ds, batch_size=Config.batch_size , shuffle=True, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:59:33.042631Z","iopub.execute_input":"2023-06-10T06:59:33.043352Z","iopub.status.idle":"2023-06-10T06:59:33.049941Z","shell.execute_reply.started":"2023-06-10T06:59:33.043307Z","shell.execute_reply":"2023-06-10T06:59:33.048786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(2):\n    for batch in tqdm(train_dl):\n        pass","metadata":{"execution":{"iopub.status.busy":"2023-06-10T06:59:33.054156Z","iopub.execute_input":"2023-06-10T06:59:33.054579Z","iopub.status.idle":"2023-06-10T07:01:11.543244Z","shell.execute_reply.started":"2023-06-10T06:59:33.054549Z","shell.execute_reply":"2023-06-10T07:01:11.541686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# JPG","metadata":{}},{"cell_type":"code","source":"class ContrailsDatasetJPEG(torch.utils.data.Dataset):\n    def __init__(self, df, train=True):\n        self.df = df\n        self.trn = train\n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        path_image = row.path_image\n        img = torchvision.io.read_image(str(path_image))\n        \n        path_mask = row.path_mask\n        label = torchvision.io.read_image(str(path_mask))\n        \n        img = img.permute(2, 0, 1).float() / 255.\n        label = label.unsqueeze(0).float()      \n            \n        return img, label\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T07:01:11.546007Z","iopub.execute_input":"2023-06-10T07:01:11.546423Z","iopub.status.idle":"2023-06-10T07:01:11.557696Z","shell.execute_reply.started":"2023-06-10T07:01:11.546378Z","shell.execute_reply":"2023-06-10T07:01:11.55649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import dataframes\ntrain_df = pd.read_csv(Paths.train_path)\nvalid_df = pd.read_csv(Paths.valid_path)\n\ntrain_df['path_image'] = Paths.pngs + train_df['record_id'].astype(str) + \"_image.jpeg\"\ntrain_df['path_mask'] = Paths.pngs + train_df['record_id'].astype(str)  + \"_mask.jpeg\"\n\n\ntrain_ds = ContrailsDatasetJPEG(\n        train_df,\n        train=True\n    )\n\ntrain_dl = DataLoader(train_ds, batch_size=Config.batch_size , shuffle=True, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2023-06-10T07:01:11.559557Z","iopub.execute_input":"2023-06-10T07:01:11.560347Z","iopub.status.idle":"2023-06-10T07:01:11.666396Z","shell.execute_reply.started":"2023-06-10T07:01:11.560305Z","shell.execute_reply":"2023-06-10T07:01:11.665179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(2):\n    for batch in tqdm(train_dl):\n        pass","metadata":{"execution":{"iopub.status.busy":"2023-06-10T07:04:42.155513Z","iopub.execute_input":"2023-06-10T07:04:42.156051Z","iopub.status.idle":"2023-06-10T07:08:18.631886Z","shell.execute_reply.started":"2023-06-10T07:04:42.156009Z","shell.execute_reply":"2023-06-10T07:08:18.630086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}