{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# READ ALL LIBRARIES WE'LL USE\n\nimport gc\n\nimport os\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tifffile as tifi\nfrom tqdm import tqdm\n\nfrom tensorflow.keras.models import load_model\nfrom PIL import Image\nimport IPython\n\nimport tensorflow as tf\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:17:54.016802Z","iopub.execute_input":"2022-09-03T21:17:54.017162Z","iopub.status.idle":"2022-09-03T21:18:01.893398Z","shell.execute_reply.started":"2022-09-03T21:17:54.017074Z","shell.execute_reply":"2022-09-03T21:18:01.892248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DOUBLE-CHECK WHETHER THE GPU IS ACTIVATED OR NOT\n\nprint(\"--> Checking for GPU\")\nfor device in tf.config.list_physical_devices():\n    print(\": {}\".format(device.name))","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:18:08.554727Z","iopub.execute_input":"2022-09-03T21:18:08.555197Z","iopub.status.idle":"2022-09-03T21:18:08.56695Z","shell.execute_reply.started":"2022-09-03T21:18:08.555153Z","shell.execute_reply":"2022-09-03T21:18:08.565811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ THE TEST META-DATA AND ADD A COLUMN WITH THE DIRECTORY PATH FOR EACH IMAGE\n\ntest_directory = os.path.join(\"../input/mayo-clinic-strip-ai/\", \"test/\")\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ntest_df[\"image_path\"] = test_df[\"image_id\"].apply(lambda x: os.path.join(test_directory, x+\".tif\"))\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:18:10.190229Z","iopub.execute_input":"2022-09-03T21:18:10.190685Z","iopub.status.idle":"2022-09-03T21:18:10.231514Z","shell.execute_reply.started":"2022-09-03T21:18:10.190646Z","shell.execute_reply":"2022-09-03T21:18:10.230463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CHECK THE SIZE OF EACH IMAGE IN THE TEST FOLDER \n\nfor img_id in test_df['image_id']:\n    print(os.path.getsize(f\"../input/mayo-clinic-strip-ai/test/{img_id}.tif\"))","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:18:12.134932Z","iopub.execute_input":"2022-09-03T21:18:12.135408Z","iopub.status.idle":"2022-09-03T21:18:12.147836Z","shell.execute_reply.started":"2022-09-03T21:18:12.135369Z","shell.execute_reply":"2022-09-03T21:18:12.146766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CELL ADAPTED FROM https://www.kaggle.com/code/realneuralnetwork/coatnet-strip-ai-inference\n# 1. CREATES A 'TEST' DIRECTORY\n# 2. READS IMAGES, CHECKS THE SIZE, REDUCES THE SIZE, AND SAVES THEM (REDUCED SIZE IMAGES) IN THE DIRECTORY CREATED\n# HELPFUL BECAUSE WE DON'T KNOW THE SIZE OF THE IMAGES THEY'LL USE FOR EVALUATION.\n# THOSE IMAGES CAN BE HUGE, AND IF NOT REDUCED (AS WE DO HERE) CAN CAUSE THE ERROR 'exceeded allowed compute'\n\ntry:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\n\n\n#new_paths = []\n#for i in tqdm(range(test_df.shape[0])):\n#    img_id = test_df.iloc[i].image_id\n#    img = cv2.resize(tifi.imread(test_directory + img_id + \".tif\"), (512, 512))\n#    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n#    new_paths.append(f\"../test/{img_id}.jpg\")\n#    del img\n#    gc.collect()\n\n\nnew_paths = []\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    #print(img_id)\n    try:\n        sz = os.path.getsize(test_directory + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 10e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifi.imread(test_directory + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512, 512, 3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    new_paths.append(f\"../test/{img_id}.jpg\")\n    del img\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:18:13.586542Z","iopub.execute_input":"2022-09-03T21:18:13.586894Z","iopub.status.idle":"2022-09-03T21:18:57.189653Z","shell.execute_reply.started":"2022-09-03T21:18:13.586864Z","shell.execute_reply":"2022-09-03T21:18:57.188637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CHECK THE SIZE OF EACH IMAGE IN THE *NEW* TEST FOLDER (I.E., AFTER RESIZE)\n# THE SIZE IS SUBSTANCIALLY SMALLER. HOPEFULLY, WE SHOULDN'T HAVE ANY MEMORY ISSUES LATER.\n\nfor img_id in new_paths:\n    print(os.path.getsize(img_id))","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:18:57.191643Z","iopub.execute_input":"2022-09-03T21:18:57.192319Z","iopub.status.idle":"2022-09-03T21:18:57.198524Z","shell.execute_reply.started":"2022-09-03T21:18:57.192279Z","shell.execute_reply":"2022-09-03T21:18:57.197225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CHECK THE NEW (RESIZED) IMAGES, THEY SHOULD LOOK THE SAME (OR VERY SIMILAR) BUT SMALLER\n\n#img_tmp = tifi.imread(new_paths[0])\n#plt.imshow(img_tmp)\n\n#IPython.display.Image(filename = new_paths[1]) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD THE MODEL WE HAVE TRAINED IN OUR LOCAL MACHINE\n\nmodel = load_model('../input/trained-model/Resized_wB_tiled_weights-ResNet50_new.h5')","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:18:57.19977Z","iopub.execute_input":"2022-09-03T21:18:57.20059Z","iopub.status.idle":"2022-09-03T21:19:12.05954Z","shell.execute_reply.started":"2022-09-03T21:18:57.200556Z","shell.execute_reply":"2022-09-03T21:19:12.058527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CHECK THE MODEL\n\nmodel.summary()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-09-03T21:19:12.063345Z","iopub.execute_input":"2022-09-03T21:19:12.063746Z","iopub.status.idle":"2022-09-03T21:19:12.179636Z","shell.execute_reply.started":"2022-09-03T21:19:12.063706Z","shell.execute_reply":"2022-09-03T21:19:12.178748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# WE DON'T KNOW THE NUMBER OF INSTANCES (ROWS) THERE'LL BE IN THE DATASET THEY'LL USE FOR EVALUATION.\n# LET'S CREATE A VARIABLE THAT RECORDS THE NUMBER OF INSTANCES. THIS WILL BE USEFUL FOR THE NEXT CELL.\n\nlength = test_df.shape[0]\nlength","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:12.180537Z","iopub.execute_input":"2022-09-03T21:19:12.180917Z","iopub.status.idle":"2022-09-03T21:19:12.188494Z","shell.execute_reply.started":"2022-09-03T21:19:12.180881Z","shell.execute_reply":"2022-09-03T21:19:12.187257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# REMEMBER TO UPDATE THE SIZE PARAMETERS ACCORDING TO THE MODEL REQUIREMENTS.\n# 1. CREATE AN EMPTY LIST IN WHICH WE'LL STORE THE PREDICTIONS.\n# 2. READ THE PATHS FROM THE 'NEW_PATHS' (RESIZED) LIST WE CREATED WITH THE DIRECTORIES OF THE RESIZED IMAGES.\n# 3. OPEN THE IMAGES, RESIZE AGAIN ACCORDING TO THE SPECIFICATIONS OF THE MODEL.\n# 4. PRE-PROCESS ACCORDING TO THE SPECIFICATIONS OF THE MODEL (IN THIS CASE JUST STANDARDIZE, /255).\n# 5. MAKE PREDICTIONS FOR EACH IMAGE AND STORE THE PREDICTIONS IN THE LIST.\n# 6. DELETE THE TEMPORARY OBJECTS AND FREE MEMORY.\n# 7. IF THE OUPUT OF print(gc.collect()) IS A NUMBER, THEN IT'S FREEING MEMORY.\n#.   NOW THAT WE HAVE REDUCED THE SIZE OF THE IMAGES (SEE ABOVE) PROBABLY NOT MUCH NEEDED TO FREE MEMORY HERE, \n#.   BUT JUST IN CASE!\n# THIS WILL BE AN ITERATIVE PROCESS (LOOP) FOR EACH IMAGE IN THE TEST_DF (HERE 4; THERE'LL BE MORE IN THE EVALUATION).\n\n\nimage_size_height = 331\nimage_size_width  = 331\n\n\npreds = []\nfor img_path in tqdm(new_paths, total = length):\n    #img_tmp = tifi.imread(img_path)\n    img_tmp = cv2.imread(img_path)\n    img_tmp = cv2.resize(img_tmp, (image_size_height, image_size_width))\n    img_tmp = img_tmp/255\n    img_tmp = np.reshape(img_tmp, [1, image_size_height, image_size_width, 3])\n    \n    pred_tmp = model.predict(img_tmp)\n    \n    preds.append(pred_tmp)\n    \n    del img_tmp  # to free memory\n    del pred_tmp # to free memory\n    gc.collect() # to free memory\n    print(gc.collect())\n","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:12.189964Z","iopub.execute_input":"2022-09-03T21:19:12.191165Z","iopub.status.idle":"2022-09-03T21:19:26.229689Z","shell.execute_reply.started":"2022-09-03T21:19:12.191128Z","shell.execute_reply":"2022-09-03T21:19:26.228618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOOK AT THE LIST OF PREDICTIONS, EACH LINE FOR ONE IMAGE.\n# LINE-WISE, THEY SHOULD ADD UP TO 1.\n\npreds","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:26.230838Z","iopub.execute_input":"2022-09-03T21:19:26.231382Z","iopub.status.idle":"2022-09-03T21:19:26.239577Z","shell.execute_reply.started":"2022-09-03T21:19:26.231351Z","shell.execute_reply":"2022-09-03T21:19:26.238294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PREDICTIONS AS A DATAFRAME.\n\npreds = pd.DataFrame(np.concatenate(preds))\npreds","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:26.24324Z","iopub.execute_input":"2022-09-03T21:19:26.24391Z","iopub.status.idle":"2022-09-03T21:19:26.257913Z","shell.execute_reply.started":"2022-09-03T21:19:26.243871Z","shell.execute_reply":"2022-09-03T21:19:26.256783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. READ THE PREDICTION FILE PROVIDED.\n# 2. REPLACE THE PREDICTIONS IN THE ORIGINAL FILE FOR OUR PREDICTIONS (I.E., FROM OUR MODEL).\n\nsubmission = pd.read_csv('../input/mayo-clinic-strip-ai/sample_submission.csv')\nsubmission.CE = preds.iloc[ : , : 1]\nsubmission.LAA = preds.iloc[ : , 1: 2]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:26.259331Z","iopub.execute_input":"2022-09-03T21:19:26.259779Z","iopub.status.idle":"2022-09-03T21:19:26.281055Z","shell.execute_reply.started":"2022-09-03T21:19:26.259743Z","shell.execute_reply":"2022-09-03T21:19:26.280033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IN CASE THE DATASET USED FOR EVALUATION HAVE MORE THAN ONE PATIENT_ID PER ROW,\n# TAKE THE MEAN OF ALL patient_id.\n# REMEMBER, FROM THE INSTRUCTIONS IN THE COMPETITION: \n#       'For each patient_id in the test set, you must predict a probability for each of the two etiology classes.'\n\nsubmission = submission.groupby(\"patient_id\").mean()","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:26.282458Z","iopub.execute_input":"2022-09-03T21:19:26.283354Z","iopub.status.idle":"2022-09-03T21:19:26.29692Z","shell.execute_reply.started":"2022-09-03T21:19:26.283317Z","shell.execute_reply":"2022-09-03T21:19:26.295856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ROUND THE PREDICTIONS TO 6 DIGITS (AS SEEN IN OTHER SUBMISSIONS).\n\nsubmission = submission[[\"CE\", \"LAA\"]].round(6).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:26.298302Z","iopub.execute_input":"2022-09-03T21:19:26.29879Z","iopub.status.idle":"2022-09-03T21:19:26.312416Z","shell.execute_reply.started":"2022-09-03T21:19:26.298751Z","shell.execute_reply":"2022-09-03T21:19:26.311441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DOUBLE CHECK THE SUBMISSION OBJECT LOOKS OK.\n\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-09-03T21:19:26.31398Z","iopub.execute_input":"2022-09-03T21:19:26.314451Z","iopub.status.idle":"2022-09-03T21:19:26.327725Z","shell.execute_reply.started":"2022-09-03T21:19:26.314418Z","shell.execute_reply":"2022-09-03T21:19:26.326629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DOUBLE CHECK THE SUBMISSION OBJECT LOOKS OK.\n\n#submission.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE THE SUBMISSION OBJECT, READY TO SUBMIT!\n\nsubmission.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T23:23:46.636251Z","iopub.execute_input":"2022-08-16T23:23:46.637333Z","iopub.status.idle":"2022-08-16T23:23:46.645268Z","shell.execute_reply.started":"2022-08-16T23:23:46.637293Z","shell.execute_reply":"2022-08-16T23:23:46.644233Z"},"trusted":true},"execution_count":null,"outputs":[]}]}