{
  "id": 428673,
  "title": "notebook cannot alot more memory hence not able to do the preprocessing of the data. what am i doing wrong ?",
  "url": "/competitions/google-research-identify-contrails-reduce-global-warming/discussion/428673",
  "author_name": "ADITHYA L BHAT",
  "post_date": "2023-08-02T09:58:46.648000",
  "votes": 1,
  "comment_count": 2,
  "views": 0,
  "content": "<p>import os<br>\nfrom PIL import Image<br>\nfrom torch.utils.data import Dataset<br>\nimport numpy as np<br>\nimport cv2<br>\nimport torch<br>\nimport albumentations as A<br>\nfrom albumentations.pytorch import ToTensorV2</p>\n<p>transform = A.Compose(<br>\n        [<br>\n            A.Resize(height=64, width=64),<br>\n            A.Rotate(limit=35, p=1.0),<br>\n            A.HorizontalFlip(p=0.5),<br>\n            A.VerticalFlip(p=0.1),<br>\n            A.Normalize(<br>\n                mean=[0.0, 0.0, 0.0],<br>\n                std=[1.0, 1.0, 1.0],<br>\n                max_pixel_value=255.0,<br>\n            ),<br>\n            ToTensorV2(),<br>\n        ],<br>\n)</p>\n<p>def normalize_range(data,bounds):<br>\n        return (data - bounds[0])/(bounds[1]-bounds[0])</p>\n<p>class ContrailDataset(Dataset):<br>\n    def <strong>init</strong>(self,data_dir,operation,Transform=None):<br>\n        self.Transform=Transform<br>\n        self.data_dir=data_dir<br>\n        self.record_id=os.listdir(data_dir)<br>\n        self.out_dir=\"/kaggle/working/\"+operation+\"_images/\"<br>\n        self.mask_dir=\"/kaggle/working/\"+operation+\"_mask/\"<br>\n        #os.mkdir(self.out_dir)<br>\n        #os.mkdir(self.mask_dir)<br>\n        self.band=[\"band_15.npy\",\"band_14.npy\",\"band_11.npy\",\"human_pixel_masks.npy\",\"human_individual_masks.npy\"]</p>\n<pre><code> ():\n     (self.images)\n\n\n ():\n    _T11_BOUNDS = (, )\n    _CLOUD_TOP_TDIFF_BOUNDS = (-, )\n    _TDIFF_BOUNDS = (-, )\n\n    image_list=[]\n    mask_list=[]\n\n     i  self.record_id[:]:\n        band15=np.load(self.data_dir++i++self.band[])\n        band14=np.load(self.data_dir++i++self.band[])\n        band11=np.load(self.data_dir++i++self.band[])\n        human_pixel_mask=np.load(self.data_dir++i++self.band[])\n        human_pixel_mask = human_pixel_mask.astype(np.uint8)\n        r=normalize_range(band15-band14,_T11_BOUNDS)\n        g=normalize_range(band14-band11,_CLOUD_TOP_TDIFF_BOUNDS)\n        b=normalize_range(band14,_T11_BOUNDS)\n        final_rgb=np.stack([r,g,b],axis=)\n        final_rgb=final_rgb[..., ]\n        \n        \n        image_list.append(final_rgb)\n        mask_list.append(human_pixel_mask)\n\n\n     \n\n\n\n     image_list, mask_list\n</code></pre>\n<p>dataset=ContrailDataset(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\",<br>\n                       \"train\",transform)</p>",
  "messages": [
    {
      "id": 2376594,
      "postDate": "2023-08-06T13:45:02.093Z",
      "content": "<p><a href=\"https://www.kaggle.com/adithyalbhat\" target=\"_blank\">@adithyalbhat</a> are you trying to load the entire data at once?</p>\n<p>I tried that initially as well, the dataset is too huge to fit in the RAM when we load it all at once. Try loading it one at a time.</p>\n<p>Your get_item will look like this</p>\n<pre><code> ():\n   \n    input_image , mask\n</code></pre>",
      "rawMarkdown": "@adithyalbhat are you trying to load the entire data at once?\n\nI tried that initially as well, the dataset is too huge to fit in the RAM when we load it all at once. Try loading it one at a time.\n\nYour get_item will look like this\n\n```python\ndef get_item(self, record_id):\n   # read one record\n   return input_image , mask\n```\n",
      "replies": [
        {
          "id": 2376596,
          "postDate": "2023-08-06T13:47:26.767Z",
          "content": "<p>Alternatively, you can use this preprocessed dataset <a href=\"https://www.kaggle.com/datasets/shashwatraman/contrails-images-ash-color\" target=\"_blank\">https://www.kaggle.com/datasets/shashwatraman/contrails-images-ash-color</a> </p>",
          "rawMarkdown": "Alternatively, you can use this preprocessed dataset https://www.kaggle.com/datasets/shashwatraman/contrails-images-ash-color "
        }
      ]
    },
    {
      "id": 2370297,
      "postDate": "2023-08-02T09:58:46.650Z",
      "content": "<p>import os<br>\nfrom PIL import Image<br>\nfrom torch.utils.data import Dataset<br>\nimport numpy as np<br>\nimport cv2<br>\nimport torch<br>\nimport albumentations as A<br>\nfrom albumentations.pytorch import ToTensorV2</p>\n<p>transform = A.Compose(<br>\n        [<br>\n            A.Resize(height=64, width=64),<br>\n            A.Rotate(limit=35, p=1.0),<br>\n            A.HorizontalFlip(p=0.5),<br>\n            A.VerticalFlip(p=0.1),<br>\n            A.Normalize(<br>\n                mean=[0.0, 0.0, 0.0],<br>\n                std=[1.0, 1.0, 1.0],<br>\n                max_pixel_value=255.0,<br>\n            ),<br>\n            ToTensorV2(),<br>\n        ],<br>\n)</p>\n<p>def normalize_range(data,bounds):<br>\n        return (data - bounds[0])/(bounds[1]-bounds[0])</p>\n<p>class ContrailDataset(Dataset):<br>\n    def <strong>init</strong>(self,data_dir,operation,Transform=None):<br>\n        self.Transform=Transform<br>\n        self.data_dir=data_dir<br>\n        self.record_id=os.listdir(data_dir)<br>\n        self.out_dir=\"/kaggle/working/\"+operation+\"_images/\"<br>\n        self.mask_dir=\"/kaggle/working/\"+operation+\"_mask/\"<br>\n        #os.mkdir(self.out_dir)<br>\n        #os.mkdir(self.mask_dir)<br>\n        self.band=[\"band_15.npy\",\"band_14.npy\",\"band_11.npy\",\"human_pixel_masks.npy\",\"human_individual_masks.npy\"]</p>\n<pre><code> ():\n     (self.images)\n\n\n ():\n    _T11_BOUNDS = (, )\n    _CLOUD_TOP_TDIFF_BOUNDS = (-, )\n    _TDIFF_BOUNDS = (-, )\n\n    image_list=[]\n    mask_list=[]\n\n     i  self.record_id[:]:\n        band15=np.load(self.data_dir++i++self.band[])\n        band14=np.load(self.data_dir++i++self.band[])\n        band11=np.load(self.data_dir++i++self.band[])\n        human_pixel_mask=np.load(self.data_dir++i++self.band[])\n        human_pixel_mask = human_pixel_mask.astype(np.uint8)\n        r=normalize_range(band15-band14,_T11_BOUNDS)\n        g=normalize_range(band14-band11,_CLOUD_TOP_TDIFF_BOUNDS)\n        b=normalize_range(band14,_T11_BOUNDS)\n        final_rgb=np.stack([r,g,b],axis=)\n        final_rgb=final_rgb[..., ]\n        \n        \n        image_list.append(final_rgb)\n        mask_list.append(human_pixel_mask)\n\n\n     \n\n\n\n     image_list, mask_list\n</code></pre>\n<p>dataset=ContrailDataset(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\",<br>\n                       \"train\",transform)</p>",
      "rawMarkdown": "import os\nfrom PIL import Image\nfrom torch.utils.data import Dataset\nimport numpy as np\nimport cv2\nimport torch\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ntransform = A.Compose(\n        [\n            A.Resize(height=64, width=64),\n            A.Rotate(limit=35, p=1.0),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.1),\n            A.Normalize(\n                mean=[0.0, 0.0, 0.0],\n                std=[1.0, 1.0, 1.0],\n                max_pixel_value=255.0,\n            ),\n            ToTensorV2(),\n        ],\n)\n\ndef normalize_range(data,bounds):\n        return (data - bounds[0])/(bounds[1]-bounds[0])\n\nclass ContrailDataset(Dataset):\n    def __init__(self,data_dir,operation,Transform=None):\n        self.Transform=Transform\n        self.data_dir=data_dir\n        self.record_id=os.listdir(data_dir)\n        self.out_dir=\"/kaggle/working/\"+operation+\"_images/\"\n        self.mask_dir=\"/kaggle/working/\"+operation+\"_mask/\"\n        #os.mkdir(self.out_dir)\n        #os.mkdir(self.mask_dir)\n        self.band=[\"band_15.npy\",\"band_14.npy\",\"band_11.npy\",\"human_pixel_masks.npy\",\"human_individual_masks.npy\"]\n        \n\n    def __len__(self):\n        return len(self.images)\n    \n\n    def __getitem__(self):\n        _T11_BOUNDS = (243, 303)\n        _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n        _TDIFF_BOUNDS = (-4, 2)\n\n        image_list=[]\n        mask_list=[]\n        \n        for i in self.record_id[:3000]:\n            band15=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[0])\n            band14=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[1])\n            band11=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[2])\n            human_pixel_mask=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[3])\n            human_pixel_mask = human_pixel_mask.astype(np.uint8)\n            r=normalize_range(band15-band14,_T11_BOUNDS)\n            g=normalize_range(band14-band11,_CLOUD_TOP_TDIFF_BOUNDS)\n            b=normalize_range(band14,_T11_BOUNDS)\n            final_rgb=np.stack([r,g,b],axis=2)\n            final_rgb=final_rgb[..., 4]\n            #cv2.imwrite(self.out_dir+i+'.jpg',final_rgb)\n            #cv2.imwrite(self.mask_dir+i+'human_pixel_mask.gif',human_pixel_mask)\n            image_list.append(final_rgb)\n            mask_list.append(human_pixel_mask)\n            \n\n        \"\"\" \n           if self.Transform is not None:\n                augmentations = self.Transform(image=final_rgb, mask=human_pixel_mask)\n                augmented_image = augmentations[\"image\"]\n                augmented_mask = augmentations[\"mask\"]\n                image_list.append(augmented_image)\n                mask_list.append(augmented_mask)\"\"\" \n                \n                \n\n        return image_list, mask_list\ndataset=ContrailDataset(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\",\n                       \"train\",transform)\n\n\n"
    }
  ],
  "comments": [
    {
      "id": 2376594,
      "author_name": "mayurimk",
      "author_url": "",
      "post_date": "2023-08-06T13:45:02.093000",
      "content": "<p><a href=\"https://www.kaggle.com/adithyalbhat\" target=\"_blank\">@adithyalbhat</a> are you trying to load the entire data at once?</p>\n<p>I tried that initially as well, the dataset is too huge to fit in the RAM when we load it all at once. Try loading it one at a time.</p>\n<p>Your get_item will look like this</p>\n<pre><code> ():\n   \n    input_image , mask\n</code></pre>",
      "votes": 0,
      "replies": [
        {
          "id": 2376596,
          "author_name": "mayurimk",
          "author_url": "",
          "post_date": "2023-08-06T13:47:26.767000",
          "content": "<p>Alternatively, you can use this preprocessed dataset <a href=\"https://www.kaggle.com/datasets/shashwatraman/contrails-images-ash-color\" target=\"_blank\">https://www.kaggle.com/datasets/shashwatraman/contrails-images-ash-color</a> </p>",
          "votes": 0,
          "replies": []
        }
      ]
    }
  ],
  "raw_markdown_by_id": {
    "2376594": "@adithyalbhat are you trying to load the entire data at once?\n\nI tried that initially as well, the dataset is too huge to fit in the RAM when we load it all at once. Try loading it one at a time.\n\nYour get_item will look like this\n\n```python\ndef get_item(self, record_id):\n   # read one record\n   return input_image , mask\n```\n",
    "2370297": "import os\nfrom PIL import Image\nfrom torch.utils.data import Dataset\nimport numpy as np\nimport cv2\nimport torch\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ntransform = A.Compose(\n        [\n            A.Resize(height=64, width=64),\n            A.Rotate(limit=35, p=1.0),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.1),\n            A.Normalize(\n                mean=[0.0, 0.0, 0.0],\n                std=[1.0, 1.0, 1.0],\n                max_pixel_value=255.0,\n            ),\n            ToTensorV2(),\n        ],\n)\n\ndef normalize_range(data,bounds):\n        return (data - bounds[0])/(bounds[1]-bounds[0])\n\nclass ContrailDataset(Dataset):\n    def __init__(self,data_dir,operation,Transform=None):\n        self.Transform=Transform\n        self.data_dir=data_dir\n        self.record_id=os.listdir(data_dir)\n        self.out_dir=\"/kaggle/working/\"+operation+\"_images/\"\n        self.mask_dir=\"/kaggle/working/\"+operation+\"_mask/\"\n        #os.mkdir(self.out_dir)\n        #os.mkdir(self.mask_dir)\n        self.band=[\"band_15.npy\",\"band_14.npy\",\"band_11.npy\",\"human_pixel_masks.npy\",\"human_individual_masks.npy\"]\n        \n\n    def __len__(self):\n        return len(self.images)\n    \n\n    def __getitem__(self):\n        _T11_BOUNDS = (243, 303)\n        _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n        _TDIFF_BOUNDS = (-4, 2)\n\n        image_list=[]\n        mask_list=[]\n        \n        for i in self.record_id[:3000]:\n            band15=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[0])\n            band14=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[1])\n            band11=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[2])\n            human_pixel_mask=np.load(self.data_dir+\"/\"+i+\"/\"+self.band[3])\n            human_pixel_mask = human_pixel_mask.astype(np.uint8)\n            r=normalize_range(band15-band14,_T11_BOUNDS)\n            g=normalize_range(band14-band11,_CLOUD_TOP_TDIFF_BOUNDS)\n            b=normalize_range(band14,_T11_BOUNDS)\n            final_rgb=np.stack([r,g,b],axis=2)\n            final_rgb=final_rgb[..., 4]\n            #cv2.imwrite(self.out_dir+i+'.jpg',final_rgb)\n            #cv2.imwrite(self.mask_dir+i+'human_pixel_mask.gif',human_pixel_mask)\n            image_list.append(final_rgb)\n            mask_list.append(human_pixel_mask)\n            \n\n        \"\"\" \n           if self.Transform is not None:\n                augmentations = self.Transform(image=final_rgb, mask=human_pixel_mask)\n                augmented_image = augmentations[\"image\"]\n                augmented_mask = augmentations[\"mask\"]\n                image_list.append(augmented_image)\n                mask_list.append(augmented_mask)\"\"\" \n                \n                \n\n        return image_list, mask_list\ndataset=ContrailDataset(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\",\n                       \"train\",transform)\n\n\n"
  }
}