{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Include related","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg\n!pip install -U pylibjpeg-libjpeg -v\n!pip install pylibjpeg pylibjpeg-libjpeg pylibjpeg-openjpeg\n!pip install pydicom\n\n# import include nesscessary\nimport cv2\nfrom PIL import Image, ImageOps\nimport argparse\nimport os\nimport glob\nimport matplotlib.pyplot as plt\nimport multiprocessing as mp\nfrom joblib import Parallel, delayed\nimport pydicom\nimport time\nimport numpy as np\nimport torch\nimport random\nimport pandas as pd\nfrom pydicom.pixel_data_handlers import apply_windowing\nimport imageio\nimport time","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:25:41.199121Z","iopub.execute_input":"2023-03-25T06:25:41.199639Z","iopub.status.idle":"2023-03-25T06:26:26.29737Z","shell.execute_reply.started":"2023-03-25T06:25:41.199604Z","shell.execute_reply":"2023-03-25T06:26:26.295104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Processing Data","metadata":{}},{"cell_type":"code","source":"from scipy import stats as sst","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:26:26.299868Z","iopub.execute_input":"2023-03-25T06:26:26.301137Z","iopub.status.idle":"2023-03-25T06:26:26.538303Z","shell.execute_reply.started":"2023-03-25T06:26:26.301078Z","shell.execute_reply":"2023-03-25T06:26:26.537174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom scipy import stats as sst\ndef cat_tia_row(img, res):\n    ## find sum pixel min col\n    img_ori = res\n#     img = img/255\n    nb_row  = img.shape[0]\n    ls = []\n    for i in range(nb_row):\n        x = np.sum(img[i, :])\n        ls.append(x)\n    ## xet left right\n    vl = 5\n    y1 = nb_row\n    y2 = 0\n    for i in range(int(nb_row/2), nb_row):\n        if ls[i] <= vl:\n            y1 = i\n            break\n    for i in reversed(range(int(nb_row/2))):\n        if ls[i] <= vl:\n            y2 = i\n            break\n    return img_ori[y2: y1, :]\ndef cat_tia_col(img, res):\n    ## find sum pixel min col\n    img_ori = res\n#     img = img/255\n    vl = np.sum(img[:, 0])\n    nb_col  = img.shape[1]\n    ls = []\n    for i in range(nb_col):\n        x = np.sum(img[:, i])\n        vl = min(vl, x)\n        ls.append(x)\n#     print(ls)\n    if vl > img.shape[0]/3: \n#         print(\"No crop col\")\n        return img_ori\n    ## xet left right\n\n    ## vì sẽ có các giá trị tương tự nhau nên ta sẽ cộn một lượng phụ trợ vào vl\n    vl += 10\n    if (np.sum(img[:, 0: int(nb_col/2) ]) > np.sum(img[:, int(nb_col/2)+1 :  ])):\n        L = 0\n        for i in range(int(nb_col/4), nb_col):\n            if ls[i] <= vl:\n#                 print(f'Col = {i}, vl = {ls[i]}')\n                img_ori = img_ori[:, : i + 50]\n                return img_ori\n    else :\n        L = 1\n        lens =  list(range(nb_col))[int(nb_col/4): ]\n        for i in reversed(lens):\n            if ls[i] <= vl:\n#                 print(f'Col = {i}, vl = {ls[i]}')\n                img_ori = img_ori[:, i-50: ]\n                return img_ori\n                \n    return img_ori       \n## show hist \ndef process_crop(img):\n    # if len(img.shape) == 3: gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    # else: gray = img\n    # gray =  cv2.GaussianBlur(gray, (7, 7), 0)\n    # # img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]\n    # image_gray = cv2.threshold(gray, 30, 255, cv2.THRESH_BINARY)[1]\n    \n    ####\n#     print(img.shape)\n    if len(img.shape)==3: \n        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    else : gray = img\n            \n    gray = cv2.GaussianBlur(gray, (7, 7), 0)\n    level = sst.trim_mean(gray.flatten(), 0.1) \n    image_gray = cv2.threshold(gray, level, 255, cv2.THRESH_BINARY)[1]\n    \n    imgcp = image_gray/255\n    h, w = image_gray.shape\n    if((h*w)-10 <= np.sum(imgcp)):\n\n        image_gray = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]\n    \n    if((h*w)/2 >= np.sum(imgcp)):\n        img = cat_tia_col(imgcp, img)\n#         print(img)\n        img = cat_tia_row(imgcp, img)\n    img = cv2.resize(img, (512, 1024))\n    try:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    except:\n        img = img\n    return img\nclass Process_Data():\n    def __init__(self, df_main, head = 'train_images' ):\n        self.df_main = df_main\n        self.path = None\n        self.head = head\n        self.id = None\n        \n    def set_path(self, id):\n#         print(len(self.df_main.patient_id))\n#         print(f'id = {id}')\n        path = f'/kaggle/input/rsna-breast-cancer-detection/train_images/{self.df_main.patient_id[id]}/{self.df_main.image_id[id]}.dcm'\n        self.path = path\n        self.id = id\n        return path\n        \n    def find_id_max(self, numLabels, labels, stats, centroids, thresh):\n        idmax = 0\n        total = np.sum(thresh)\n        sum_pixel = total/4\n\n        density_ar = total/(thresh.shape[0]*thresh.shape[1]);\n        for i in range(0, numLabels): \n            x = stats[i, cv2.CC_STAT_LEFT] \n            y = stats[i, cv2.CC_STAT_TOP] \n            w = stats[i, cv2.CC_STAT_WIDTH] \n            h = stats[i, cv2.CC_STAT_HEIGHT] \n\n            sums = np.sum(thresh[y:y+h, x : x+ w])\n            density_e = sums/ ((y+h)* (x+w))\n            if sum_pixel <= sums and density_e >= density_ar :\n                idmax = i\n                density_ar  = density_e\n        return idmax\n        \n    def ConnectedComponents(self, image, size = (1024, 1024, 3) ):\n        expen = 100\n        if len(image.shape)==3: gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n        else : gray = image\n            \n        gray = cv2.GaussianBlur(gray, (7, 7), 0)\n        level = sst.trim_mean(gray.flatten(), 0.1) \n        thresh = cv2.threshold(gray, level, 255, cv2.THRESH_BINARY)[1]\n\n        numLabels, labels, stats, centroids = cv2.connectedComponentsWithStats(\n                        thresh,\n                        4,\n                        cv2.CV_32S\n        )\n        idmax = self.find_id_max(numLabels, labels, stats, centroids, thresh)\n        x = stats[idmax, cv2.CC_STAT_LEFT] \n        y = stats[idmax, cv2.CC_STAT_TOP] \n        w = stats[idmax, cv2.CC_STAT_WIDTH] \n        h = stats[idmax, cv2.CC_STAT_HEIGHT] \n\n        w_max =  image.shape[1]\n        h_max =  image.shape[0] - 1\n\n        thresh = thresh/255\n        if (np.sum(thresh[:, 0: int(w_max/2) ]) < np.sum(thresh[:, int(w_max/2)+1 :  ])):\n\n            new_image = image[y: y+h , max(0,x-expen) : ]\n        else :\n            new_image = image[y: y+h , : min(w_max, x + w+ expen)]\n        new_image = cv2.resize(new_image, size)\n        return new_image\n    \n    def procees_dcm(self):\n\n        dicom = pydicom.dcmread(self.path)\n#         try : \n        img = dicom.pixel_array\n    \n            \n        img = apply_windowing(img, dicom)\n        img = (img - img.min()) / (img.max() - img.min())\n\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":  \n            img = 1 - img\n        image = np.uint8(img * 255)\n        return image\n    def resize_img(self, img):\n        img = cv2.resize(img, (1024, 1024))\n        try:\n            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n        except:\n            img = img\n        return img\n    def get_info(self):\n        \n        id =self.id\n        id_img = self.df_main['image_id'][id]\n        \n        id_par = self.df_main.patient_id[id]\n        view = self.df_main.view[id]\n        cancer = 1\n        return [id_img, id_par, view, view]\n    def save_image(self, img):\n        ls = self.get_info()\n        file_name = f'/kaggle/working/{self.head}/{ls[2]}_{ls[0]}.png';\n        imageio.imwrite(file_name, img)\n        \n    def crop_image(self, size=(1024, 1024, 3)):\n        new_image = self.procees_dcm()\n        new_image = self.ConnectedComponents(new_image)\n        self.save_image(new_image)\n        return new_image\n    def crop_image_for_you(self, size=(1024, 1024, 3)):\n        # Reading the dicom file and converting it to a numpy array.\n        new_image = self.procees_dcm()\n        # new_image = self.ConnectedComponents(new_image)\n#         new_image = cv2.imread(self.path)\n#         print(self.path)\n        new_image = self.resize_img(new_image)\n        \n        self.save_image(new_image)\n        return new_image\n    \n    # def show(self, cl = 'gray'):\n    #     fig, axs = plt.subplots(1, 2)\n    #     origin_img = self.procees_dcm()\n    #     start = time.time()\n    #     croped_img = self.crop_image(self.path)\n    #     end = time.time()\n    #     file_name = self.path.split('/')\n    #     id_img = int(file_name[-1].split('.')[0])\n    #     target = {0: 'no-cancer', 1: 'cancer'}\n    #     info = self.get_info()\n    #     fig.suptitle(f'Id_img: {info[0]} \\n Id_par: Id_par: {info[1]} \\n State: {target[info[3]]} \\n Views: {info[2]} \\n Time = {end - start} \\n\\n')\n    #     axs[0].imshow(origin_img, cmap='bone')\n    #     axs[0].set_title('Origin')\n    #     axs[1].imshow(croped_img, cmap='bone')\n    #     axs[1].set_title('Croped image')\n    def process1(self, i):\n        self.set_path(i)\n        self.crop_image_for_you()\n    def my_method(self, ls):\n#         for i in ls:\n#             print(i)\n        with mp.Pool(12) as p:\n            p.map(self.process1, ls)\n#             self.process1(i)\n    \n            \n        \n","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:38:17.039809Z","iopub.execute_input":"2023-03-25T06:38:17.040275Z","iopub.status.idle":"2023-03-25T06:38:17.080483Z","shell.execute_reply.started":"2023-03-25T06:38:17.040238Z","shell.execute_reply":"2023-03-25T06:38:17.079372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n# data_train[data_train.image_id == 1091299886]","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:38:17.624971Z","iopub.execute_input":"2023-03-25T06:38:17.625684Z","iopub.status.idle":"2023-03-25T06:38:17.679796Z","shell.execute_reply.started":"2023-03-25T06:38:17.625637Z","shell.execute_reply":"2023-03-25T06:38:17.678253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:38:21.056807Z","iopub.execute_input":"2023-03-25T06:38:21.057173Z","iopub.status.idle":"2023-03-25T06:38:21.084556Z","shell.execute_reply.started":"2023-03-25T06:38:21.057146Z","shell.execute_reply":"2023-03-25T06:38:21.083406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_cancer = data_train[data_train.cancer == 1]\n# len(data_cancer)\n# data_cancer\ndata_train = data_train[data_train.view.isin(['CC', 'MLO'])]\ndata_train","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:38:22.229231Z","iopub.execute_input":"2023-03-25T06:38:22.23006Z","iopub.status.idle":"2023-03-25T06:38:22.26219Z","shell.execute_reply.started":"2023-03-25T06:38:22.23003Z","shell.execute_reply":"2023-03-25T06:38:22.260728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# img = cv2.imread('/kaggle/working/CC_541722628.png')\n# img.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:28:20.156354Z","iopub.execute_input":"2023-03-25T06:28:20.157034Z","iopub.status.idle":"2023-03-25T06:28:20.163241Z","shell.execute_reply.started":"2023-03-25T06:28:20.157003Z","shell.execute_reply":"2023-03-25T06:28:20.16121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = \"0-20000\"","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:39:14.059009Z","iopub.execute_input":"2023-03-25T06:39:14.05937Z","iopub.status.idle":"2023-03-25T06:39:14.06476Z","shell.execute_reply.started":"2023-03-25T06:39:14.059337Z","shell.execute_reply":"2023-03-25T06:39:14.063361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(f'/kaggle/working/{name}/')","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:39:14.588811Z","iopub.execute_input":"2023-03-25T06:39:14.58977Z","iopub.status.idle":"2023-03-25T06:39:14.596098Z","shell.execute_reply.started":"2023-03-25T06:39:14.589696Z","shell.execute_reply":"2023-03-25T06:39:14.594727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pro1 = Process_Data(data_train, name)\nls =  list(data_train.index)\n# pro1.process1(18366)\npro1.my_method(ls[20000:40000])","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:39:15.538741Z","iopub.execute_input":"2023-03-25T06:39:15.539136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train[data_train['image_id'].isin([737468199,802890376, 1386883888] )]","metadata":{"execution":{"iopub.status.busy":"2023-03-25T03:10:00.07891Z","iopub.execute_input":"2023-03-25T03:10:00.07934Z","iopub.status.idle":"2023-03-25T03:10:00.106405Z","shell.execute_reply.started":"2023-03-25T03:10:00.0793Z","shell.execute_reply":"2023-03-25T03:10:00.104458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show dicom","metadata":{}},{"cell_type":"code","source":"# path = '/kaggle/input/rsna-breast-cancer-detection/train_images/45629/1386883888.dcm'\n# ds = pydicom.dcmread(path)\n\n# plt.imshow(ds.pixel_array)","metadata":{"execution":{"iopub.status.busy":"2023-03-08T06:53:50.572344Z","iopub.execute_input":"2023-03-08T06:53:50.573166Z","iopub.status.idle":"2023-03-08T06:53:51.642854Z","shell.execute_reply.started":"2023-03-08T06:53:50.573119Z","shell.execute_reply":"2023-03-08T06:53:51.641557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-03-08T03:38:28.670588Z","iopub.execute_input":"2023-03-08T03:38:28.67105Z","iopub.status.idle":"2023-03-08T03:38:28.749902Z","shell.execute_reply.started":"2023-03-08T03:38:28.671012Z","shell.execute_reply":"2023-03-08T03:38:28.748526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DDSM","metadata":{}},{"cell_type":"code","source":"# ddsm = pd.read_csv('/kaggle/input/ddsm-mammography-positive-case/additional_train.csv')\n# ddsm['image_id'] = list(range(len(ddsm)))","metadata":{"execution":{"iopub.status.busy":"2023-03-07T00:11:06.042636Z","iopub.execute_input":"2023-03-07T00:11:06.043881Z","iopub.status.idle":"2023-03-07T00:11:06.057197Z","shell.execute_reply.started":"2023-03-07T00:11:06.043813Z","shell.execute_reply":"2023-03-07T00:11:06.055857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pro1 = Process_Data(ddsm)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-07T00:13:47.022292Z","iopub.execute_input":"2023-03-07T00:13:47.022729Z","iopub.status.idle":"2023-03-07T00:13:47.028501Z","shell.execute_reply.started":"2023-03-07T00:13:47.022692Z","shell.execute_reply":"2023-03-07T00:13:47.026972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def process(i):\n#     pro1.set_path(i)\n#     pro1.crop_image()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T00:15:37.022134Z","iopub.execute_input":"2023-03-07T00:15:37.022559Z","iopub.status.idle":"2023-03-07T00:15:37.029297Z","shell.execute_reply.started":"2023-03-07T00:15:37.02252Z","shell.execute_reply":"2023-03-07T00:15:37.027766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ls = list(range(len(ddsm)))â\n# with mp.Pool(12) as p:\n#         p.map(process,  ls)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T00:54:01.016529Z","iopub.execute_input":"2023-03-07T00:54:01.017123Z","iopub.status.idle":"2023-03-07T00:58:24.241111Z","shell.execute_reply.started":"2023-03-07T00:54:01.017077Z","shell.execute_reply":"2023-03-07T00:58:24.237445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# shutil.rmtree(\"/kaggle/working/\")","metadata":{"execution":{"iopub.status.busy":"2023-03-25T06:39:05.170224Z","iopub.execute_input":"2023-03-25T06:39:05.170584Z","iopub.status.idle":"2023-03-25T06:39:05.192305Z","shell.execute_reply.started":"2023-03-25T06:39:05.17055Z","shell.execute_reply":"2023-03-25T06:39:05.191191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with mp.Pool(12) as p:\n#         p.map(crop_image,  ls)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T15:16:59.56166Z","iopub.status.idle":"2023-03-06T15:16:59.563051Z","shell.execute_reply.started":"2023-03-06T15:16:59.562792Z","shell.execute_reply":"2023-03-06T15:16:59.562822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# os.chdir(r'/kaggle/working/')\n# from IPython.display import FileLink\n# !zip -r file.zip '/kaggle/working/'\n# FileLink(r'file.zip')","metadata":{"execution":{"iopub.status.busy":"2023-03-06T08:50:51.592031Z","iopub.execute_input":"2023-03-06T08:50:51.593115Z","iopub.status.idle":"2023-03-06T08:50:52.888357Z","shell.execute_reply.started":"2023-03-06T08:50:51.593066Z","shell.execute_reply":"2023-03-06T08:50:52.885911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset and DataLoader","metadata":{}},{"cell_type":"code","source":"# import os\n# import pandas as pd\n# from torchvision.io import read_image","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:49.913935Z","iopub.execute_input":"2023-03-04T08:40:49.914218Z","iopub.status.idle":"2023-03-04T08:40:50.138919Z","shell.execute_reply.started":"2023-03-04T08:40:49.914191Z","shell.execute_reply":"2023-03-04T08:40:50.137898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torchvision\n# from torchvision import transforms","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.140423Z","iopub.execute_input":"2023-03-04T08:40:50.140719Z","iopub.status.idle":"2023-03-04T08:40:50.145483Z","shell.execute_reply.started":"2023-03-04T08:40:50.140691Z","shell.execute_reply":"2023-03-04T08:40:50.144082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transform1 = transforms.Compose([\n#                     transforms.ToTensor(),\n#                     transforms.Resize((224, 224)),\n#                     transforms.Normalize([0.5], [0.5]),\n#                     transforms.RandomHorizontalFlip(p=0.5),\n#                     transforms.RandomRotation(10)]\n    \n# )\n# transform2 = transforms.Compose([\n#                     transforms.ToTensor(),\n#                     transforms.Resize((224, 224)),\n#                     transforms.Normalize([0.5], [0.5])]\n    \n# )","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.146942Z","iopub.execute_input":"2023-03-04T08:40:50.147313Z","iopub.status.idle":"2023-03-04T08:40:50.15598Z","shell.execute_reply.started":"2023-03-04T08:40:50.147275Z","shell.execute_reply":"2023-03-04T08:40:50.155303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Dataset_for_RSNA():\n#     def __init__(self, data, transform=True, augment = True, type_data = 'RNSA'):\n#         self.type = type_data\n#         self.data = data\n#         self.process = Process_Data(self.data)\n#         self.transform = transform\n#         self.augment = augment\n#     def __len__(self):\n#         return len(self.data)\n#     def __getitem__(self, idx):\n        \n#         self.process.set_path(idx)\n#         img = self.process.crop_image()\n#         label = self.process.get_info()[-1]\n#         # Train for train set\n#         if self.augment and label == 1:\n#             img = transform1(img)\n#         else : \n#             # Train for train val, test\n#             img = transform2(img)\n            \n#         return img, label\n        ","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.158942Z","iopub.execute_input":"2023-03-04T08:40:50.159237Z","iopub.status.idle":"2023-03-04T08:40:50.168017Z","shell.execute_reply.started":"2023-03-04T08:40:50.159209Z","shell.execute_reply":"2023-03-04T08:40:50.167028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_set = data_train.sample(frac=0.7, random_state = 42).reset_index(drop=True)\n# train_val = data_train.sample(frac=0.2, random_state = 42).reset_index(drop=True)\n# train_test = data_train.sample(frac=0.1, random_state = 42).reset_index(drop=True)\n# print(train_set.shape[0], train_val.shape, train_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.16936Z","iopub.execute_input":"2023-03-04T08:40:50.169913Z","iopub.status.idle":"2023-03-04T08:40:50.195571Z","shell.execute_reply.started":"2023-03-04T08:40:50.169879Z","shell.execute_reply":"2023-03-04T08:40:50.194283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_set","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.19673Z","iopub.execute_input":"2023-03-04T08:40:50.198984Z","iopub.status.idle":"2023-03-04T08:40:50.221381Z","shell.execute_reply.started":"2023-03-04T08:40:50.198931Z","shell.execute_reply":"2023-03-04T08:40:50.220198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # train_dataset = Dataset_for_RSNA(train_set, transform=True, augment = True)\n# val_dataset = Dataset_for_RSNA(train_val, transform=True, augment = False)\n# test_dataset = Dataset_for_RSNA(train_test, transform=True, augment = False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.222648Z","iopub.execute_input":"2023-03-04T08:40:50.222937Z","iopub.status.idle":"2023-03-04T08:40:50.231079Z","shell.execute_reply.started":"2023-03-04T08:40:50.222909Z","shell.execute_reply":"2023-03-04T08:40:50.230342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Show image\n+ Tại sao chúng ta không thể sử dụng hàm show như bình thường trong pytorch\nVì định dạng ảnh trong pytorch nó theo dạng là (batch_size, channels, height, width) trong đó batch_size là số lượng ảnh trong một lô\nTuy nhiên, Matplotlib yêu cầu ảnh có dạng (height, width, channels) để hiển thị chính vì vậy chúng ta phải sử dụng torch.squeeze() để loại bỏ các đối số của pytorch và giữ lại các tính chất của hình ảnh","metadata":{}},{"cell_type":"code","source":"# # for i in range(100):\n# #     ls = random.randint(0, train_set.shape[0])\n# # ls = 1\n# img, label = train_dataset.__getitem__(20)\n\n# plt.imshow(img.squeeze(), cmap='gray')\n# plt.title(f'{label}')\n# plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:40:50.231929Z","iopub.execute_input":"2023-03-04T08:40:50.232216Z","iopub.status.idle":"2023-03-04T08:40:51.295787Z","shell.execute_reply.started":"2023-03-04T08:40:50.232186Z","shell.execute_reply":"2023-03-04T08:40:51.294774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch_size = 64\n\n# train_dataloader = torch.utils.data.DataLoader(train_set, batch_size = batch_size, shuffle=True)\n# val_dataloader = torch.utils.data.DataLoader(train_val, batch_size = batch_size, shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2023-03-04T08:43:46.535659Z","iopub.execute_input":"2023-03-04T08:43:46.536033Z","iopub.status.idle":"2023-03-04T08:43:46.541889Z","shell.execute_reply.started":"2023-03-04T08:43:46.535998Z","shell.execute_reply":"2023-03-04T08:43:46.540761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Giải thích:\n+ torch.utils.data.DataLoader là một lớp trong PyTorch cung cấp một iterable cho một tập dữ liệu PyTorch được cung cấp. Nó cho phép bạn tải một tập dữ liệu thành các batch và cung cấp một số tùy chọn để tùy chỉnh quá trình tải dữ liệu.\n+ Train_set: là bộ dữ liệu của mình\n+ batch_size là kích thước của mỗi nhóm, ví dụ như ta có 1000 mẫu thì chia mỗi batch là 100 thì sẽ xử lý 1000 mẫu cùng một lúc\n+ shuffle: True nếu muốn trộn mẫu","metadata":{}},{"cell_type":"markdown","source":"## DataLoader","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}