{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA TFWriter\n\nThe following is an approach to writing TFRecords to a Bucket in GCS.\n\n# Imports/Setup","metadata":{"_uuid":"d7133559-9970-4d9f-90be-fd9214475242","_cell_guid":"ee427b66-3b68-48c6-bbf1-52cff2ba1c11","trusted":true}},{"cell_type":"code","source":"!pip install python-gdcm -q\n!pip install pylibjpeg -q","metadata":{"_uuid":"353a9a95-4e32-4ceb-83c2-946067272193","_cell_guid":"36248c92-661d-47ce-93d3-4deac0f59630","collapsed":false,"scrolled":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:27:42.829774Z","iopub.execute_input":"2023-02-18T18:27:42.830254Z","iopub.status.idle":"2023-02-18T18:28:04.96775Z","shell.execute_reply.started":"2023-02-18T18:27:42.830218Z","shell.execute_reply":"2023-02-18T18:28:04.966497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport pydicom\nimport os\nimport cv2\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow import keras\nassert tf.__version__ >= \"2.11\"\nfrom google.cloud import storage\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_uuid":"b46f5278-1071-42b1-813a-00c127e12d03","_cell_guid":"98a71bff-b9b1-4901-8107-b0d040ca6af8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:04.970038Z","iopub.execute_input":"2023-02-18T18:28:04.971076Z","iopub.status.idle":"2023-02-18T18:28:04.978824Z","shell.execute_reply.started":"2023-02-18T18:28:04.971026Z","shell.execute_reply":"2023-02-18T18:28:04.977726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.version.VERSION","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:04.9804Z","iopub.execute_input":"2023-02-18T18:28:04.980741Z","iopub.status.idle":"2023-02-18T18:28:04.994512Z","shell.execute_reply.started":"2023-02-18T18:28:04.980706Z","shell.execute_reply":"2023-02-18T18:28:04.993447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_test = '/kaggle/input/rsna-breast-cancer-detection/test.csv'\npath_train = '/kaggle/input/rsna-breast-cancer-detection/train.csv'","metadata":{"_uuid":"854fbbe6-7ed1-47d4-b46f-14cf582cc8a8","_cell_guid":"c2c2ddf1-adeb-4416-a286-4492f2b48ff8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:04.997256Z","iopub.execute_input":"2023-02-18T18:28:04.997626Z","iopub.status.idle":"2023-02-18T18:28:05.005826Z","shell.execute_reply.started":"2023-02-18T18:28:04.997593Z","shell.execute_reply":"2023-02-18T18:28:05.004875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfte = pd.read_csv(path_test)\ndftr = pd.read_csv(path_train)","metadata":{"_uuid":"fac19436-dc92-4d83-8484-976987822200","_cell_guid":"0868eb78-79fe-41f7-a96f-44eafe21abc1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:05.007253Z","iopub.execute_input":"2023-02-18T18:28:05.00767Z","iopub.status.idle":"2023-02-18T18:28:05.176541Z","shell.execute_reply.started":"2023-02-18T18:28:05.00762Z","shell.execute_reply":"2023-02-18T18:28:05.175706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Handle the Data\n\nFirst we need to split the training data into a training and validation set.\n\nTo get the image paths we can apply a function to the dataframe on each row to get the picture with patient_id and image_id.\n\nNext we do some basic preproccessing to make sure each image is normalized.","metadata":{"_uuid":"5ec75849-6b64-4947-9efb-bd1fd21db472","_cell_guid":"539c699b-325b-4fe6-a94b-fc6da83a1201","trusted":true}},{"cell_type":"code","source":"def train_test_split(dataset):\n    split = int(dataset.shape[0] * .8)\n    return dataset[:split], dataset[split:]","metadata":{"_uuid":"c435000e-768d-4f2f-a508-2fc14c274187","_cell_guid":"e428fd9c-08e2-4cf5-b994-40fe12054a43","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:05.178057Z","iopub.execute_input":"2023-02-18T18:28:05.178719Z","iopub.status.idle":"2023-02-18T18:28:05.184869Z","shell.execute_reply.started":"2023-02-18T18:28:05.178667Z","shell.execute_reply":"2023-02-18T18:28:05.1838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_img_path_to_pd(row):\n    return '/kaggle/input/rsna-breast-cancer-detection/train_images/' + str(row['patient_id']) + '/' + str(row['image_id']) + '.dcm'","metadata":{"_uuid":"c64c54c0-2896-4889-a86a-54936e832a46","_cell_guid":"58b348d8-1425-4775-8abf-0797a8f3fcd4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:05.186093Z","iopub.execute_input":"2023-02-18T18:28:05.186477Z","iopub.status.idle":"2023-02-18T18:28:05.197289Z","shell.execute_reply.started":"2023-02-18T18:28:05.186448Z","shell.execute_reply":"2023-02-18T18:28:05.196046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image ROI","metadata":{"_uuid":"57240afa-ae57-49bf-b48b-5bf48acaff2b","_cell_guid":"4dab83af-9704-4e7e-86e4-93f82b46251a","trusted":true}},{"cell_type":"code","source":"def crop_coords(img):\n    \"\"\"\n    Crop ROI from image.\n    \"\"\"\n    # Otsu's thresholding after Gaussian filtering\n    blur = cv2.GaussianBlur(img, (5, 5), 0)\n    _, breast_mask = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)\n    \n    cnts, _ = cv2.findContours(breast_mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    cnt = max(cnts, key = cv2.contourArea)\n    x, y, w, h = cv2.boundingRect(cnt)\n    return (x, y, w, h)","metadata":{"_uuid":"16fb58fd-9893-4b52-930a-0c5513f845c5","_cell_guid":"db4e60b3-6e3b-4329-b751-25316c73a8d7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:05.198699Z","iopub.execute_input":"2023-02-18T18:28:05.199721Z","iopub.status.idle":"2023-02-18T18:28:05.210841Z","shell.execute_reply.started":"2023-02-18T18:28:05.199683Z","shell.execute_reply":"2023-02-18T18:28:05.209572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Normalize the data and clip the data to inlcude only between 5 < x < 99 percentile of data.","metadata":{"_uuid":"d424f386-c067-448f-aa38-f8ee072bd5af","_cell_guid":"d622b53f-4045-4686-b639-634bdafaa257","trusted":true}},{"cell_type":"code","source":"def truncation_normalization(img):\n    Pmin = np.percentile(img[img!=0], 5)\n    Pmax = np.percentile(img[img!=0], 99)\n    truncated = np.clip(img, Pmin, Pmax)  \n    normalized = (truncated - Pmin)/(Pmax - Pmin)\n    normalized[img==0]=0\n    return normalized","metadata":{"_uuid":"365d246c-f8d3-49e0-9624-dfbb5d950bc8","_cell_guid":"8944a287-d2c2-4535-a368-e5344a7720b7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:05.212307Z","iopub.execute_input":"2023-02-18T18:28:05.21262Z","iopub.status.idle":"2023-02-18T18:28:05.221309Z","shell.execute_reply.started":"2023-02-18T18:28:05.212592Z","shell.execute_reply":"2023-02-18T18:28:05.220352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clahe(img, clip):\n    clahe = cv2.createCLAHE(clipLimit=clip)\n    cl = clahe.apply(img)\n    return cl\n\ndef parse_clahe(image):\n    cl1 = clahe(image, 1.0)\n    cl2 = clahe(image, 2.0)\n    img_final = cv2.merge((image, cl1, cl2))\n    return img_final","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:05.225538Z","iopub.execute_input":"2023-02-18T18:28:05.225876Z","iopub.status.idle":"2023-02-18T18:28:05.23779Z","shell.execute_reply.started":"2023-02-18T18:28:05.225847Z","shell.execute_reply":"2023-02-18T18:28:05.236903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> smaller: (384, 672)","metadata":{}},{"cell_type":"code","source":"def preprocess(imagepath, voi=False):\n    ds = pydicom.dcmread(imagepath)\n    img = ds.pixel_array\n    \n    if voi:\n        img = apply_voi_lut(img, ds)\n    \n    img_max = np.max(img)\n    img = img / np.max(img)\n    if ds.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n    img = img * img_max\n    img = img[..., tf.newaxis]\n    \n    (x, y, w, h) = crop_coords(img.astype(\"uint8\"))\n    img_cropped = img[y:y+h, x:x+w]\n    \n    img_normalized = truncation_normalization(img_cropped)\n                                              \n    img_final = cv2.resize(img_normalized, (int(768/2), int(1344/2)))\n    \n    img_final = np.array(img_final*255, dtype=np.uint8)\n    img_final = img_final[..., tf.newaxis]\n    #img_final = cv2.equalizeHist(img_final)\n    \n    # img_final = parse_clahe(img_final)\n                                              \n    return img_final","metadata":{"_uuid":"7f7c0a1d-acb7-4018-b458-77c00cca83f0","_cell_guid":"db796db1-cf46-4e17-a667-a6f199cda37d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:05.239305Z","iopub.execute_input":"2023-02-18T18:28:05.239598Z","iopub.status.idle":"2023-02-18T18:28:05.249703Z","shell.execute_reply.started":"2023-02-18T18:28:05.239572Z","shell.execute_reply":"2023-02-18T18:28:05.24868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dftr[dftr['cancer'] == 1]","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:05.251079Z","iopub.execute_input":"2023-02-18T18:28:05.251949Z","iopub.status.idle":"2023-02-18T18:28:05.296772Z","shell.execute_reply.started":"2023-02-18T18:28:05.251916Z","shell.execute_reply":"2023-02-18T18:28:05.295652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augm_lay = keras.Sequential(\n    [\n        keras.layers.RandomZoom(height_factor=(0, -0.3))\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:05.298263Z","iopub.execute_input":"2023-02-18T18:28:05.29901Z","iopub.status.idle":"2023-02-18T18:28:05.380808Z","shell.execute_reply.started":"2023-02-18T18:28:05.298963Z","shell.execute_reply":"2023-02-18T18:28:05.379726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_image(image, y):\n    \n    image = tf.image.random_brightness(image, 0.10)\n    # image = tf.image.random_contrast(image, 0.90, 1.40)\n    # image = tf.image.random_saturation(image, 0.50, 2.00)\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_jpeg_quality(image, 75, 100)\n    \n    image = data_augm_lay(image)\n    \n    # image = dropout(image)\n    \n    return image, y","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:05.382139Z","iopub.execute_input":"2023-02-18T18:28:05.383098Z","iopub.status.idle":"2023-02-18T18:28:05.390015Z","shell.execute_reply.started":"2023-02-18T18:28:05.383063Z","shell.execute_reply":"2023-02-18T18:28:05.388597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image1 = preprocess('/kaggle/input/rsna-breast-cancer-detection/train_images/10130/388811999.dcm', voi=False)\nplt.imshow(image1)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:05.392037Z","iopub.execute_input":"2023-02-18T18:28:05.392902Z","iopub.status.idle":"2023-02-18T18:28:06.804293Z","shell.execute_reply.started":"2023-02-18T18:28:05.392857Z","shell.execute_reply":"2023-02-18T18:28:06.803226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image2 = preprocess('/kaggle/input/rsna-breast-cancer-detection/train_images/10130/388811999.dcm', voi=True)\nplt.imshow(image2)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:06.805664Z","iopub.execute_input":"2023-02-18T18:28:06.806004Z","iopub.status.idle":"2023-02-18T18:28:08.215327Z","shell.execute_reply.started":"2023-02-18T18:28:06.805961Z","shell.execute_reply":"2023-02-18T18:28:08.21413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image1.dtype","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:08.21674Z","iopub.execute_input":"2023-02-18T18:28:08.217074Z","iopub.status.idle":"2023-02-18T18:28:08.223528Z","shell.execute_reply.started":"2023-02-18T18:28:08.217043Z","shell.execute_reply":"2023-02-18T18:28:08.222367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image3 = tf.io.encode_jpeg(image2)\nimage3 = tf.io.decode_jpeg(image3)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:08.225149Z","iopub.execute_input":"2023-02-18T18:28:08.225494Z","iopub.status.idle":"2023-02-18T18:28:08.250715Z","shell.execute_reply.started":"2023-02-18T18:28:08.225452Z","shell.execute_reply":"2023-02-18T18:28:08.249495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image3.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:08.252462Z","iopub.execute_input":"2023-02-18T18:28:08.252809Z","iopub.status.idle":"2023-02-18T18:28:08.259463Z","shell.execute_reply.started":"2023-02-18T18:28:08.252775Z","shell.execute_reply":"2023-02-18T18:28:08.258411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(image3)","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:08.262313Z","iopub.execute_input":"2023-02-18T18:28:08.262759Z","iopub.status.idle":"2023-02-18T18:28:08.510866Z","shell.execute_reply.started":"2023-02-18T18:28:08.262715Z","shell.execute_reply":"2023-02-18T18:28:08.509974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.imshow(augment_image(image2, 1)[0])","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:08.512147Z","iopub.execute_input":"2023-02-18T18:28:08.512665Z","iopub.status.idle":"2023-02-18T18:28:08.517771Z","shell.execute_reply.started":"2023-02-18T18:28:08.512628Z","shell.execute_reply":"2023-02-18T18:28:08.516486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image2.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-18T18:28:08.519384Z","iopub.execute_input":"2023-02-18T18:28:08.519774Z","iopub.status.idle":"2023-02-18T18:28:08.531294Z","shell.execute_reply.started":"2023-02-18T18:28:08.519727Z","shell.execute_reply":"2023-02-18T18:28:08.530442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Useful Stats Writing Data\n\nThis is not a EDA Notebook book but we do need some useful statistics to deal with data.","metadata":{"_uuid":"19bff1e2-1cb6-4976-bd4c-9338c630ca1c","_cell_guid":"590eac8d-6d47-4dc8-8af6-e0bc522b2495","trusted":true}},{"cell_type":"code","source":"def useful_stats():\n    print('pos train %: ' + str(dftr[dftr['cancer'] == 1].shape[0] / dftr.shape[0] * 100))\n    print('pos valid %: ' + str(dfv[dfv['cancer'] == 1].shape[0] / dfv.shape[0] * 100))","metadata":{"_uuid":"a305bb4f-c4be-4b68-ac08-2ba89bf306d2","_cell_guid":"5a5e3cc9-deb7-4ea3-8cf4-5b4e95da28e7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.532834Z","iopub.execute_input":"2023-02-18T18:28:08.533477Z","iopub.status.idle":"2023-02-18T18:28:08.542358Z","shell.execute_reply.started":"2023-02-18T18:28:08.533444Z","shell.execute_reply":"2023-02-18T18:28:08.541225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dealing With Files locally and in the Cloud\n\nWe need to follow the sequence: \n> write to local file -> push to cloud -> delete from local -> repeat","metadata":{"_uuid":"d75f328d-d9d7-4636-bcc3-5382d7ce8225","_cell_guid":"a69ba64a-dd42-4e76-8d34-b574d7ffa0bf","trusted":true}},{"cell_type":"code","source":"def push_to_cloud(filepath, bucket_name):\n    print('pushing ' + filepath + ' to cloud in bucket: train_batches')\n    \n    bucket = storage_client.get_bucket(bucket_name)\n    blob = bucket.blob(filepath)\n    blob.upload_from_filename('/kaggle/working/' + filepath)","metadata":{"_uuid":"e4b34ef5-ab3a-4489-9a51-1c96feeb2dca","_cell_guid":"9a2f799d-b1df-4c6a-8954-3720588b7451","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.544969Z","iopub.execute_input":"2023-02-18T18:28:08.545515Z","iopub.status.idle":"2023-02-18T18:28:08.554174Z","shell.execute_reply.started":"2023-02-18T18:28:08.545472Z","shell.execute_reply":"2023-02-18T18:28:08.552908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def delete_file(filepath):\n    print('deleting ' + filepath + ' from local')\n    os.remove('/kaggle/working/' + filepath)\n    \ndef clear_all_local():\n    for k in os.listdir('/kaggle/working/'):\n        if k == '.virtual_documents':\n            continue\n        delete_file(k)","metadata":{"_uuid":"f50c92ff-abae-494f-ac83-98b5ba6596d5","_cell_guid":"c9635ec1-6b90-4ecf-a8d9-b8cb923a355e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.555985Z","iopub.execute_input":"2023-02-18T18:28:08.556344Z","iopub.status.idle":"2023-02-18T18:28:08.565356Z","shell.execute_reply.started":"2023-02-18T18:28:08.556315Z","shell.execute_reply":"2023-02-18T18:28:08.564135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create an Example\n\nCreate an example using the features necessary to the Network.","metadata":{"_uuid":"d767e9a0-7b1a-44ed-b996-fd204d45b535","_cell_guid":"63cd7fb3-3a7b-4855-9cd6-ce5831b77c1c","trusted":true}},{"cell_type":"code","source":"from tensorflow.train import BytesList, FloatList, Int64List\nfrom tensorflow.train import Feature, Features, Example\n\ndef get_example(image, label, age, implant, laterality, view, diff_neg):\n    return Example(\n        features=Features(\n            feature={\n                'image': Feature(bytes_list=BytesList(value=[tf.io.encode_jpeg(image, optimize_size=True).numpy()])),\n                'label': Feature(int64_list=Int64List(value=[label])),\n                'age': Feature(int64_list=Int64List(value=[age])),\n                'impant': Feature(int64_list=Int64List(value=[implant])),\n                'laterality': Feature(bytes_list=BytesList(value=[laterality])),\n                'view': Feature(bytes_list=BytesList(value=[view])),\n                'diff_neg': Feature(int64_list=Int64List(value=[diff_neg]))\n            }\n        )\n    )","metadata":{"_uuid":"0f3e8dff-744b-442e-a734-a289996ef11c","_cell_guid":"bcfc6ae3-40f2-45e3-8a60-a8ddbe1bffea","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.567183Z","iopub.execute_input":"2023-02-18T18:28:08.567526Z","iopub.status.idle":"2023-02-18T18:28:08.57892Z","shell.execute_reply.started":"2023-02-18T18:28:08.567469Z","shell.execute_reply":"2023-02-18T18:28:08.577947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Write TFRecords\n\nTo write the files we need the dataset to read from, the name of the batches we are writing, and the bucket to write to.\n\nThe dataest is split into n batches and use offset incase connection is lost.\n\nFor every file in the n files created we create an example using the corresponding instance's values in the dataframe.\n\nFinally the example is serialized and written it to the file.\n\nThis file is then pushed to gcs and deleted from the local system.","metadata":{"_uuid":"8ea704b9-16db-4cc6-a696-ff2ecd728600","_cell_guid":"47485c1a-a73d-43a5-8943-c3c749a53ac0","trusted":true}},{"cell_type":"code","source":"def write_tfrecords(name, dataset, bucket_name, n_shards, offset):\n    paths = [\"{}_batch_{:0>3}.tfrecord\".format(name, index) for index in range(offset, n_shards)]\n    \n    for num, i in enumerate(paths):\n        with tf.io.TFRecordWriter(i) as f: \n            print('writing to ' + i )\n            for index, row in dataset[(num+offset)*(int(dataset.shape[0] / n_shards) + 1):(num+offset+1)*(int(dataset.shape[0] / n_shards) + 1)].iterrows():\n                image = preprocess(row['imagepath'], voi=True)\n                label = row['cancer']\n                age = int(row['age'])\n                implant = row['implant']\n                laterality = bytes(row['laterality'], 'utf-8')\n                view = bytes(row['view'], 'utf-8')\n                diff_neg = 1 if row['difficult_negative_case'] else 0\n                example = get_example(image, label, age, implant, laterality, view, diff_neg)\n                f.write(example.SerializeToString())\n                \n        push_to_cloud(i, bucket_name)\n        delete_file(i)\n            \n    return paths","metadata":{"_uuid":"d381d2e2-757a-429b-bcd0-c3d57874fa1f","_cell_guid":"3bfe1d2d-de54-41fa-8381-507a62452daa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.580667Z","iopub.execute_input":"2023-02-18T18:28:08.581223Z","iopub.status.idle":"2023-02-18T18:28:08.593553Z","shell.execute_reply.started":"2023-02-18T18:28:08.581179Z","shell.execute_reply":"2023-02-18T18:28:08.592495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define GCS Storage Area","metadata":{"_uuid":"74ef7322-94e5-4630-8990-6438552a9614","_cell_guid":"8c4ff856-1de7-4451-b45a-35fd6b155c1b","trusted":true}},{"cell_type":"code","source":"client_area = 'kagglersna01'\nstorage_client = storage.Client(project=client_area)","metadata":{"_uuid":"d332ccea-a99a-45fd-a505-3358a906fbe5","_cell_guid":"12b50b13-90a2-4c65-b295-29678041112f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.598305Z","iopub.execute_input":"2023-02-18T18:28:08.59863Z","iopub.status.idle":"2023-02-18T18:28:08.606204Z","shell.execute_reply.started":"2023-02-18T18:28:08.598604Z","shell.execute_reply":"2023-02-18T18:28:08.605279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apply Transformation to DataFrame","metadata":{"_uuid":"dfdf98a2-9cf5-4d57-ad7f-6bce2f0b5c0e","_cell_guid":"4031c4af-f848-4ccd-a510-98b38c161355","trusted":true}},{"cell_type":"code","source":"def setup_data(dirr, dataset):\n    dataset.fillna(dataset['age'].median(), inplace=True)\n    dataset['imagepath'] = dataset.apply(add_img_path_to_pd, axis=1)\n    return train_test_split(dataset)\ndftr, dfv = setup_data('train_images', dftr)","metadata":{"_uuid":"1cea7b28-4aee-41d0-b750-c14b7ef69afe","_cell_guid":"29495634-339f-4a8d-9078-f2a2c33b1c75","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:08.607548Z","iopub.execute_input":"2023-02-18T18:28:08.607853Z","iopub.status.idle":"2023-02-18T18:28:09.496511Z","shell.execute_reply.started":"2023-02-18T18:28:08.607825Z","shell.execute_reply":"2023-02-18T18:28:09.49532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"useful_stats()","metadata":{"_uuid":"63e23269-7fce-40e2-a7d3-a0ae4f55ab18","_cell_guid":"e9aae4ac-7cd3-48ca-a22c-fb7970c0893d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:09.497911Z","iopub.execute_input":"2023-02-18T18:28:09.498245Z","iopub.status.idle":"2023-02-18T18:28:09.512547Z","shell.execute_reply.started":"2023-02-18T18:28:09.498217Z","shell.execute_reply":"2023-02-18T18:28:09.511282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run TFRecord Writer","metadata":{"_uuid":"0ca50844-395d-4c7d-9c4e-9aef5e96b9a7","_cell_guid":"7d255ccd-793f-4d0a-b468-b4b1fc1be0ca","trusted":true}},{"cell_type":"code","source":"def main_run(name, dataset, bucket_name, n_shards=20, offset=0):\n    clear_all_local()\n    write_tfrecords(name, dataset, bucket_name, n_shards, offset)","metadata":{"_uuid":"d63c325a-40a9-4039-9452-685d090f9046","_cell_guid":"bdd42772-a58e-4594-bcdd-69b0c3cb07d4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:09.514474Z","iopub.execute_input":"2023-02-18T18:28:09.514919Z","iopub.status.idle":"2023-02-18T18:28:09.521069Z","shell.execute_reply.started":"2023-02-18T18:28:09.514877Z","shell.execute_reply":"2023-02-18T18:28:09.519641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BUCKET_NAME = 'train_batches_smaller'\n# main_run('train', dftr, BUCKET_NAME, n_shards=int(200/2), offset=0)\nmain_run('valid', dfv, BUCKET_NAME, n_shards=int(50/2), offset=8)","metadata":{"_uuid":"8fc8ffb1-a2be-440e-baa5-f05026243c9b","_cell_guid":"b20e18b8-7403-45cf-9b40-64ad675a6936","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-18T18:28:09.522568Z","iopub.execute_input":"2023-02-18T18:28:09.523558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parse an Example Instance","metadata":{"_uuid":"1af2ec9b-d997-4a0c-8784-4828768cb699","_cell_guid":"1ea7ae2a-f9da-4d25-9ebe-950954ab0e8d","trusted":true}},{"cell_type":"code","source":"def parse_example(tfrecord):\n    feature_desc = {\n        'image': tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n        'label': tf.io.FixedLenFeature([], tf.int64, default_value=-1),\n        'age': tf.io.FixedLenFeature([], tf.int64, default_value=-1),\n        'impant': tf.io.FixedLenFeature([], tf.int64, default_value=-1),\n        'laterality': tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n        'view': tf.io.FixedLenFeature([], tf.string, default_value=\"\"),\n        'diff_neg': tf.io.FixedLenFeature([], tf.int64, default_value=-1)\n    }\n    \n    example = tf.io.parse_single_example(tfrecord, feature_desc)\n    image = tf.io.decode_jpeg(example[\"image\"], channels=1)\n    image = tf.reshape(image, shape=[768/2, 1344/2, 1])\n    return image, tf.cast(example[\"label\"], tf.float32)","metadata":{"_uuid":"80819fdf-6e69-47af-bd9a-f7480f411b13","_cell_guid":"644762ca-ce2f-4d15-937a-86707aa3ed63","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Dataset From GCS Files\n\nCreate a tf.dataset from the file names in gcs.\n\nWe use multi-threading to speed up processing time.\n\nWe also shuffle the dataset to add more variablility. \n\nWe call dataset.map() to parse each example in the file.","metadata":{"_uuid":"131c1730-30f9-47b1-a266-bf29136f8c5b","_cell_guid":"137a46aa-e5df-40a6-aa07-835ba6332761","trusted":true}},{"cell_type":"code","source":"def record_dataset(filepaths, shuffle_buffer_size=5000, batch_size=32, training=True, ordered=False):\n    \n    ignore_order = tf.data.Options()\n    \n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = tf.data.TFRecordDataset(filepaths, num_parallel_reads=tf.data.AUTOTUNE)\n    \n    #dataset = dataset.cache()\n    \n    dataset = dataset.map(parse_example, num_parallel_calls=tf.data.AUTOTUNE)\n    \n    if training:\n        #dataset = dataset.filter(undersample_majority)\n        #dataset = dataset.map(augment_image, num_parallel_calls=tf.data.AUTOTUNE)\n        dataset = dataset.with_options(ignore_order)\n        #dataset = dataset.shuffle(shuffle_buffer_size)\n        dataset = dataset.repeat()\n        \n    dataset = dataset.batch(batch_size)\n    \n    dataset = dataset.prefetch(tf.data.AUTOTUNE)\n    return dataset","metadata":{"_uuid":"088b5d05-c3d9-4a53-8196-8fa8366760d3","_cell_guid":"aab30f62-77ba-4999-ba70-c67fa1ffae7d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainpaths = ['gs://' + 'train_batches' + '/' + \"{}_batch_{:0>3}.tfrecord\".format('train', index) for index in range(0, 200)]\nvalidpaths = ['gs://' + 'train_batches' + '/' + \"{}_batch_{:0>3}.tfrecord\".format('valid', index) for index in range(0, 50)]\n\ntrain_set = record_dataset(trainpaths, batch_size=64)\nvalid_set = record_dataset(validpaths, batch_size=64, training=False)","metadata":{"_uuid":"42cbf9d8-f625-482a-8077-abf902b6687e","_cell_guid":"d44a84f8-66b3-4ace-a7e2-cf32ed4b3c14","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_set:\n    for k in i[:64]:\n        print(k.shape)\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}