{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Embolism Detection EDA"},{"metadata":{},"cell_type":"markdown","source":"This is an EDA (exploratory data analysis) for the newly launched RNSA **Embolism Detection** competition on Kaggle that we're going to be working on today. An **embolism** is caused when your arteries are blocked off in your lung, preventing blood flow and stopping your lung from getting the oxygen it needs to carry out respiration. It is the most fatal cardiovascular disease in the United States of America (60,000 to 100,000 deaths per annum)."},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y\nimport gdcm\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nturbo_colormap_data = np.array(\n                       [[0.18995,0.07176,0.23217],\n                       [0.19483,0.08339,0.26149],\n                       [0.19956,0.09498,0.29024],\n                       [0.20415,0.10652,0.31844],\n                       [0.20860,0.11802,0.34607],\n                       [0.21291,0.12947,0.37314],\n                       [0.21708,0.14087,0.39964],\n                       [0.22111,0.15223,0.42558],\n                       [0.22500,0.16354,0.45096],\n                       [0.22875,0.17481,0.47578],\n                       [0.23236,0.18603,0.50004],\n                       [0.23582,0.19720,0.52373],\n                       [0.23915,0.20833,0.54686],\n                       [0.24234,0.21941,0.56942],\n                       [0.24539,0.23044,0.59142],\n                       [0.24830,0.24143,0.61286],\n                       [0.25107,0.25237,0.63374],\n                       [0.25369,0.26327,0.65406],\n                       [0.25618,0.27412,0.67381],\n                       [0.25853,0.28492,0.69300],\n                       [0.26074,0.29568,0.71162],\n                       [0.26280,0.30639,0.72968],\n                       [0.26473,0.31706,0.74718],\n                       [0.26652,0.32768,0.76412],\n                       [0.26816,0.33825,0.78050],\n                       [0.26967,0.34878,0.79631],\n                       [0.27103,0.35926,0.81156],\n                       [0.27226,0.36970,0.82624],\n                       [0.27334,0.38008,0.84037],\n                       [0.27429,0.39043,0.85393],\n                       [0.27509,0.40072,0.86692],\n                       [0.27576,0.41097,0.87936],\n                       [0.27628,0.42118,0.89123],\n                       [0.27667,0.43134,0.90254],\n                       [0.27691,0.44145,0.91328],\n                       [0.27701,0.45152,0.92347],\n                       [0.27698,0.46153,0.93309],\n                       [0.27680,0.47151,0.94214],\n                       [0.27648,0.48144,0.95064],\n                       [0.27603,0.49132,0.95857],\n                       [0.27543,0.50115,0.96594],\n                       [0.27469,0.51094,0.97275],\n                       [0.27381,0.52069,0.97899],\n                       [0.27273,0.53040,0.98461],\n                       [0.27106,0.54015,0.98930],\n                       [0.26878,0.54995,0.99303],\n                       [0.26592,0.55979,0.99583],\n                       [0.26252,0.56967,0.99773],\n                       [0.25862,0.57958,0.99876],\n                       [0.25425,0.58950,0.99896],\n                       [0.24946,0.59943,0.99835],\n                       [0.24427,0.60937,0.99697],\n                       [0.23874,0.61931,0.99485],\n                       [0.23288,0.62923,0.99202],\n                       [0.22676,0.63913,0.98851],\n                       [0.22039,0.64901,0.98436],\n                       [0.21382,0.65886,0.97959],\n                       [0.20708,0.66866,0.97423],\n                       [0.20021,0.67842,0.96833],\n                       [0.19326,0.68812,0.96190],\n                       [0.18625,0.69775,0.95498],\n                       [0.17923,0.70732,0.94761],\n                       [0.17223,0.71680,0.93981],\n                       [0.16529,0.72620,0.93161],\n                       [0.15844,0.73551,0.92305],\n                       [0.15173,0.74472,0.91416],\n                       [0.14519,0.75381,0.90496],\n                       [0.13886,0.76279,0.89550],\n                       [0.13278,0.77165,0.88580],\n                       [0.12698,0.78037,0.87590],\n                       [0.12151,0.78896,0.86581],\n                       [0.11639,0.79740,0.85559],\n                       [0.11167,0.80569,0.84525],\n                       [0.10738,0.81381,0.83484],\n                       [0.10357,0.82177,0.82437],\n                       [0.10026,0.82955,0.81389],\n                       [0.09750,0.83714,0.80342],\n                       [0.09532,0.84455,0.79299],\n                       [0.09377,0.85175,0.78264],\n                       [0.09287,0.85875,0.77240],\n                       [0.09267,0.86554,0.76230],\n                       [0.09320,0.87211,0.75237],\n                       [0.09451,0.87844,0.74265],\n                       [0.09662,0.88454,0.73316],\n                       [0.09958,0.89040,0.72393],\n                       [0.10342,0.89600,0.71500],\n                       [0.10815,0.90142,0.70599],\n                       [0.11374,0.90673,0.69651],\n                       [0.12014,0.91193,0.68660],\n                       [0.12733,0.91701,0.67627],\n                       [0.13526,0.92197,0.66556],\n                       [0.14391,0.92680,0.65448],\n                       [0.15323,0.93151,0.64308],\n                       [0.16319,0.93609,0.63137],\n                       [0.17377,0.94053,0.61938],\n                       [0.18491,0.94484,0.60713],\n                       [0.19659,0.94901,0.59466],\n                       [0.20877,0.95304,0.58199],\n                       [0.22142,0.95692,0.56914],\n                       [0.23449,0.96065,0.55614],\n                       [0.24797,0.96423,0.54303],\n                       [0.26180,0.96765,0.52981],\n                       [0.27597,0.97092,0.51653],\n                       [0.29042,0.97403,0.50321],\n                       [0.30513,0.97697,0.48987],\n                       [0.32006,0.97974,0.47654],\n                       [0.33517,0.98234,0.46325],\n                       [0.35043,0.98477,0.45002],\n                       [0.36581,0.98702,0.43688],\n                       [0.38127,0.98909,0.42386],\n                       [0.39678,0.99098,0.41098],\n                       [0.41229,0.99268,0.39826],\n                       [0.42778,0.99419,0.38575],\n                       [0.44321,0.99551,0.37345],\n                       [0.45854,0.99663,0.36140],\n                       [0.47375,0.99755,0.34963],\n                       [0.48879,0.99828,0.33816],\n                       [0.50362,0.99879,0.32701],\n                       [0.51822,0.99910,0.31622],\n                       [0.53255,0.99919,0.30581],\n                       [0.54658,0.99907,0.29581],\n                       [0.56026,0.99873,0.28623],\n                       [0.57357,0.99817,0.27712],\n                       [0.58646,0.99739,0.26849],\n                       [0.59891,0.99638,0.26038],\n                       [0.61088,0.99514,0.25280],\n                       [0.62233,0.99366,0.24579],\n                       [0.63323,0.99195,0.23937],\n                       [0.64362,0.98999,0.23356],\n                       [0.65394,0.98775,0.22835],\n                       [0.66428,0.98524,0.22370],\n                       [0.67462,0.98246,0.21960],\n                       [0.68494,0.97941,0.21602],\n                       [0.69525,0.97610,0.21294],\n                       [0.70553,0.97255,0.21032],\n                       [0.71577,0.96875,0.20815],\n                       [0.72596,0.96470,0.20640],\n                       [0.73610,0.96043,0.20504],\n                       [0.74617,0.95593,0.20406],\n                       [0.75617,0.95121,0.20343],\n                       [0.76608,0.94627,0.20311],\n                       [0.77591,0.94113,0.20310],\n                       [0.78563,0.93579,0.20336],\n                       [0.79524,0.93025,0.20386],\n                       [0.80473,0.92452,0.20459],\n                       [0.81410,0.91861,0.20552],\n                       [0.82333,0.91253,0.20663],\n                       [0.83241,0.90627,0.20788],\n                       [0.84133,0.89986,0.20926],\n                       [0.85010,0.89328,0.21074],\n                       [0.85868,0.88655,0.21230],\n                       [0.86709,0.87968,0.21391],\n                       [0.87530,0.87267,0.21555],\n                       [0.88331,0.86553,0.21719],\n                       [0.89112,0.85826,0.21880],\n                       [0.89870,0.85087,0.22038],\n                       [0.90605,0.84337,0.22188],\n                       [0.91317,0.83576,0.22328],\n                       [0.92004,0.82806,0.22456],\n                       [0.92666,0.82025,0.22570],\n                       [0.93301,0.81236,0.22667],\n                       [0.93909,0.80439,0.22744],\n                       [0.94489,0.79634,0.22800],\n                       [0.95039,0.78823,0.22831],\n                       [0.95560,0.78005,0.22836],\n                       [0.96049,0.77181,0.22811],\n                       [0.96507,0.76352,0.22754],\n                       [0.96931,0.75519,0.22663],\n                       [0.97323,0.74682,0.22536],\n                       [0.97679,0.73842,0.22369],\n                       [0.98000,0.73000,0.22161],\n                       [0.98289,0.72140,0.21918],\n                       [0.98549,0.71250,0.21650],\n                       [0.98781,0.70330,0.21358],\n                       [0.98986,0.69382,0.21043],\n                       [0.99163,0.68408,0.20706],\n                       [0.99314,0.67408,0.20348],\n                       [0.99438,0.66386,0.19971],\n                       [0.99535,0.65341,0.19577],\n                       [0.99607,0.64277,0.19165],\n                       [0.99654,0.63193,0.18738],\n                       [0.99675,0.62093,0.18297],\n                       [0.99672,0.60977,0.17842],\n                       [0.99644,0.59846,0.17376],\n                       [0.99593,0.58703,0.16899],\n                       [0.99517,0.57549,0.16412],\n                       [0.99419,0.56386,0.15918],\n                       [0.99297,0.55214,0.15417],\n                       [0.99153,0.54036,0.14910],\n                       [0.98987,0.52854,0.14398],\n                       [0.98799,0.51667,0.13883],\n                       [0.98590,0.50479,0.13367],\n                       [0.98360,0.49291,0.12849],\n                       [0.98108,0.48104,0.12332],\n                       [0.97837,0.46920,0.11817],\n                       [0.97545,0.45740,0.11305],\n                       [0.97234,0.44565,0.10797],\n                       [0.96904,0.43399,0.10294],\n                       [0.96555,0.42241,0.09798],\n                       [0.96187,0.41093,0.09310],\n                       [0.95801,0.39958,0.08831],\n                       [0.95398,0.38836,0.08362],\n                       [0.94977,0.37729,0.07905],\n                       [0.94538,0.36638,0.07461],\n                       [0.94084,0.35566,0.07031],\n                       [0.93612,0.34513,0.06616],\n                       [0.93125,0.33482,0.06218],\n                       [0.92623,0.32473,0.05837],\n                       [0.92105,0.31489,0.05475],\n                       [0.91572,0.30530,0.05134],\n                       [0.91024,0.29599,0.04814],\n                       [0.90463,0.28696,0.04516],\n                       [0.89888,0.27824,0.04243],\n                       [0.89298,0.26981,0.03993],\n                       [0.88691,0.26152,0.03753],\n                       [0.88066,0.25334,0.03521],\n                       [0.87422,0.24526,0.03297],\n                       [0.86760,0.23730,0.03082],\n                       [0.86079,0.22945,0.02875],\n                       [0.85380,0.22170,0.02677],\n                       [0.84662,0.21407,0.02487],\n                       [0.83926,0.20654,0.02305],\n                       [0.83172,0.19912,0.02131],\n                       [0.82399,0.19182,0.01966],\n                       [0.81608,0.18462,0.01809],\n                       [0.80799,0.17753,0.01660],\n                       [0.79971,0.17055,0.01520],\n                       [0.79125,0.16368,0.01387],\n                       [0.78260,0.15693,0.01264],\n                       [0.77377,0.15028,0.01148],\n                       [0.76476,0.14374,0.01041],\n                       [0.75556,0.13731,0.00942],\n                       [0.74617,0.13098,0.00851],\n                       [0.73661,0.12477,0.00769],\n                       [0.72686,0.11867,0.00695],\n                       [0.71692,0.11268,0.00629],\n                       [0.70680,0.10680,0.00571],\n                       [0.69650,0.10102,0.00522],\n                       [0.68602,0.09536,0.00481],\n                       [0.67535,0.08980,0.00449],\n                       [0.66449,0.08436,0.00424],\n                       [0.65345,0.07902,0.00408],\n                       [0.64223,0.07380,0.00401],\n                       [0.63082,0.06868,0.00401],\n                       [0.61923,0.06367,0.00410],\n                       [0.60746,0.05878,0.00427],\n                       [0.59550,0.05399,0.00453],\n                       [0.58336,0.04931,0.00486],\n                       [0.57103,0.04474,0.00529],\n                       [0.55852,0.04028,0.00579],\n                       [0.54583,0.03593,0.00638],\n                       [0.53295,0.03169,0.00705],\n                       [0.51989,0.02756,0.00780],\n                       [0.50664,0.02354,0.00863],\n                       [0.49321,0.01963,0.00955],\n                       [0.47960,0.01583,0.01055]])\n\n\n\n\ndef RGBToPyCmap(rgbdata):\n    nsteps = rgbdata.shape[0]\n    stepaxis = np.linspace(0, 1, nsteps)\n\n    rdata=[]; gdata=[]; bdata=[]\n    for istep in range(nsteps):\n        r = rgbdata[istep,0]\n        g = rgbdata[istep,1]\n        b = rgbdata[istep,2]\n        rdata.append((stepaxis[istep], r, r))\n        gdata.append((stepaxis[istep], g, g))\n        bdata.append((stepaxis[istep], b, b))\n\n    mpl_data = {'red':   rdata,\n                 'green': gdata,\n                 'blue':  bdata}\n\n    return mpl_data\n\n\nmpl_data = RGBToPyCmap(turbo_colormap_data)\nplt.register_cmap(name='turbo', data=mpl_data, lut=turbo_colormap_data.shape[0])\n\nmpl_data_r = RGBToPyCmap(turbo_colormap_data[::-1,:])\nplt.register_cmap(name='turbo_r', data=mpl_data_r, lut=turbo_colormap_data.shape[0])\n\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nturbo_colormap_data = np.array(\n                       [[0.18995,0.07176,0.23217],\n                       [0.19483,0.08339,0.26149],\n                       [0.19956,0.09498,0.29024],\n                       [0.20415,0.10652,0.31844],\n                       [0.20860,0.11802,0.34607],\n                       [0.21291,0.12947,0.37314],\n                       [0.21708,0.14087,0.39964],\n                       [0.22111,0.15223,0.42558],\n                       [0.22500,0.16354,0.45096],\n                       [0.22875,0.17481,0.47578],\n                       [0.23236,0.18603,0.50004],\n                       [0.23582,0.19720,0.52373],\n                       [0.23915,0.20833,0.54686],\n                       [0.24234,0.21941,0.56942],\n                       [0.24539,0.23044,0.59142],\n                       [0.24830,0.24143,0.61286],\n                       [0.25107,0.25237,0.63374],\n                       [0.25369,0.26327,0.65406],\n                       [0.25618,0.27412,0.67381],\n                       [0.25853,0.28492,0.69300],\n                       [0.26074,0.29568,0.71162],\n                       [0.26280,0.30639,0.72968],\n                       [0.26473,0.31706,0.74718],\n                       [0.26652,0.32768,0.76412],\n                       [0.26816,0.33825,0.78050],\n                       [0.26967,0.34878,0.79631],\n                       [0.27103,0.35926,0.81156],\n                       [0.27226,0.36970,0.82624],\n                       [0.27334,0.38008,0.84037],\n                       [0.27429,0.39043,0.85393],\n                       [0.27509,0.40072,0.86692],\n                       [0.27576,0.41097,0.87936],\n                       [0.27628,0.42118,0.89123],\n                       [0.27667,0.43134,0.90254],\n                       [0.27691,0.44145,0.91328],\n                       [0.27701,0.45152,0.92347],\n                       [0.27698,0.46153,0.93309],\n                       [0.27680,0.47151,0.94214],\n                       [0.27648,0.48144,0.95064],\n                       [0.27603,0.49132,0.95857],\n                       [0.27543,0.50115,0.96594],\n                       [0.27469,0.51094,0.97275],\n                       [0.27381,0.52069,0.97899],\n                       [0.27273,0.53040,0.98461],\n                       [0.27106,0.54015,0.98930],\n                       [0.26878,0.54995,0.99303],\n                       [0.26592,0.55979,0.99583],\n                       [0.26252,0.56967,0.99773],\n                       [0.25862,0.57958,0.99876],\n                       [0.25425,0.58950,0.99896],\n                       [0.24946,0.59943,0.99835],\n                       [0.24427,0.60937,0.99697],\n                       [0.23874,0.61931,0.99485],\n                       [0.23288,0.62923,0.99202],\n                       [0.22676,0.63913,0.98851],\n                       [0.22039,0.64901,0.98436],\n                       [0.21382,0.65886,0.97959],\n                       [0.20708,0.66866,0.97423],\n                       [0.20021,0.67842,0.96833],\n                       [0.19326,0.68812,0.96190],\n                       [0.18625,0.69775,0.95498],\n                       [0.17923,0.70732,0.94761],\n                       [0.17223,0.71680,0.93981],\n                       [0.16529,0.72620,0.93161],\n                       [0.15844,0.73551,0.92305],\n                       [0.15173,0.74472,0.91416],\n                       [0.14519,0.75381,0.90496],\n                       [0.13886,0.76279,0.89550],\n                       [0.13278,0.77165,0.88580],\n                       [0.12698,0.78037,0.87590],\n                       [0.12151,0.78896,0.86581],\n                       [0.11639,0.79740,0.85559],\n                       [0.11167,0.80569,0.84525],\n                       [0.10738,0.81381,0.83484],\n                       [0.10357,0.82177,0.82437],\n                       [0.10026,0.82955,0.81389],\n                       [0.09750,0.83714,0.80342],\n                       [0.09532,0.84455,0.79299],\n                       [0.09377,0.85175,0.78264],\n                       [0.09287,0.85875,0.77240],\n                       [0.09267,0.86554,0.76230],\n                       [0.09320,0.87211,0.75237],\n                       [0.09451,0.87844,0.74265],\n                       [0.09662,0.88454,0.73316],\n                       [0.09958,0.89040,0.72393],\n                       [0.10342,0.89600,0.71500],\n                       [0.10815,0.90142,0.70599],\n                       [0.11374,0.90673,0.69651],\n                       [0.12014,0.91193,0.68660],\n                       [0.12733,0.91701,0.67627],\n                       [0.13526,0.92197,0.66556],\n                       [0.14391,0.92680,0.65448],\n                       [0.15323,0.93151,0.64308],\n                       [0.16319,0.93609,0.63137],\n                       [0.17377,0.94053,0.61938],\n                       [0.18491,0.94484,0.60713],\n                       [0.19659,0.94901,0.59466],\n                       [0.20877,0.95304,0.58199],\n                       [0.22142,0.95692,0.56914],\n                       [0.23449,0.96065,0.55614],\n                       [0.24797,0.96423,0.54303],\n                       [0.26180,0.96765,0.52981],\n                       [0.27597,0.97092,0.51653],\n                       [0.29042,0.97403,0.50321],\n                       [0.30513,0.97697,0.48987],\n                       [0.32006,0.97974,0.47654],\n                       [0.33517,0.98234,0.46325],\n                       [0.35043,0.98477,0.45002],\n                       [0.36581,0.98702,0.43688],\n                       [0.38127,0.98909,0.42386],\n                       [0.39678,0.99098,0.41098],\n                       [0.41229,0.99268,0.39826],\n                       [0.42778,0.99419,0.38575],\n                       [0.44321,0.99551,0.37345],\n                       [0.45854,0.99663,0.36140],\n                       [0.47375,0.99755,0.34963],\n                       [0.48879,0.99828,0.33816],\n                       [0.50362,0.99879,0.32701],\n                       [0.51822,0.99910,0.31622],\n                       [0.53255,0.99919,0.30581],\n                       [0.54658,0.99907,0.29581],\n                       [0.56026,0.99873,0.28623],\n                       [0.57357,0.99817,0.27712],\n                       [0.58646,0.99739,0.26849],\n                       [0.59891,0.99638,0.26038],\n                       [0.61088,0.99514,0.25280],\n                       [0.62233,0.99366,0.24579],\n                       [0.63323,0.99195,0.23937],\n                       [0.64362,0.98999,0.23356],\n                       [0.65394,0.98775,0.22835],\n                       [0.66428,0.98524,0.22370],\n                       [0.67462,0.98246,0.21960],\n                       [0.68494,0.97941,0.21602],\n                       [0.69525,0.97610,0.21294],\n                       [0.70553,0.97255,0.21032],\n                       [0.71577,0.96875,0.20815],\n                       [0.72596,0.96470,0.20640],\n                       [0.73610,0.96043,0.20504],\n                       [0.74617,0.95593,0.20406],\n                       [0.75617,0.95121,0.20343],\n                       [0.76608,0.94627,0.20311],\n                       [0.77591,0.94113,0.20310],\n                       [0.78563,0.93579,0.20336],\n                       [0.79524,0.93025,0.20386],\n                       [0.80473,0.92452,0.20459],\n                       [0.81410,0.91861,0.20552],\n                       [0.82333,0.91253,0.20663],\n                       [0.83241,0.90627,0.20788],\n                       [0.84133,0.89986,0.20926],\n                       [0.85010,0.89328,0.21074],\n                       [0.85868,0.88655,0.21230],\n                       [0.86709,0.87968,0.21391],\n                       [0.87530,0.87267,0.21555],\n                       [0.88331,0.86553,0.21719],\n                       [0.89112,0.85826,0.21880],\n                       [0.89870,0.85087,0.22038],\n                       [0.90605,0.84337,0.22188],\n                       [0.91317,0.83576,0.22328],\n                       [0.92004,0.82806,0.22456],\n                       [0.92666,0.82025,0.22570],\n                       [0.93301,0.81236,0.22667],\n                       [0.93909,0.80439,0.22744],\n                       [0.94489,0.79634,0.22800],\n                       [0.95039,0.78823,0.22831],\n                       [0.95560,0.78005,0.22836],\n                       [0.96049,0.77181,0.22811],\n                       [0.96507,0.76352,0.22754],\n                       [0.96931,0.75519,0.22663],\n                       [0.97323,0.74682,0.22536],\n                       [0.97679,0.73842,0.22369],\n                       [0.98000,0.73000,0.22161],\n                       [0.98289,0.72140,0.21918],\n                       [0.98549,0.71250,0.21650],\n                       [0.98781,0.70330,0.21358],\n                       [0.98986,0.69382,0.21043],\n                       [0.99163,0.68408,0.20706],\n                       [0.99314,0.67408,0.20348],\n                       [0.99438,0.66386,0.19971],\n                       [0.99535,0.65341,0.19577],\n                       [0.99607,0.64277,0.19165],\n                       [0.99654,0.63193,0.18738],\n                       [0.99675,0.62093,0.18297],\n                       [0.99672,0.60977,0.17842],\n                       [0.99644,0.59846,0.17376],\n                       [0.99593,0.58703,0.16899],\n                       [0.99517,0.57549,0.16412],\n                       [0.99419,0.56386,0.15918],\n                       [0.99297,0.55214,0.15417],\n                       [0.99153,0.54036,0.14910],\n                       [0.98987,0.52854,0.14398],\n                       [0.98799,0.51667,0.13883],\n                       [0.98590,0.50479,0.13367],\n                       [0.98360,0.49291,0.12849],\n                       [0.98108,0.48104,0.12332],\n                       [0.97837,0.46920,0.11817],\n                       [0.97545,0.45740,0.11305],\n                       [0.97234,0.44565,0.10797],\n                       [0.96904,0.43399,0.10294],\n                       [0.96555,0.42241,0.09798],\n                       [0.96187,0.41093,0.09310],\n                       [0.95801,0.39958,0.08831],\n                       [0.95398,0.38836,0.08362],\n                       [0.94977,0.37729,0.07905],\n                       [0.94538,0.36638,0.07461],\n                       [0.94084,0.35566,0.07031],\n                       [0.93612,0.34513,0.06616],\n                       [0.93125,0.33482,0.06218],\n                       [0.92623,0.32473,0.05837],\n                       [0.92105,0.31489,0.05475],\n                       [0.91572,0.30530,0.05134],\n                       [0.91024,0.29599,0.04814],\n                       [0.90463,0.28696,0.04516],\n                       [0.89888,0.27824,0.04243],\n                       [0.89298,0.26981,0.03993],\n                       [0.88691,0.26152,0.03753],\n                       [0.88066,0.25334,0.03521],\n                       [0.87422,0.24526,0.03297],\n                       [0.86760,0.23730,0.03082],\n                       [0.86079,0.22945,0.02875],\n                       [0.85380,0.22170,0.02677],\n                       [0.84662,0.21407,0.02487],\n                       [0.83926,0.20654,0.02305],\n                       [0.83172,0.19912,0.02131],\n                       [0.82399,0.19182,0.01966],\n                       [0.81608,0.18462,0.01809],\n                       [0.80799,0.17753,0.01660],\n                       [0.79971,0.17055,0.01520],\n                       [0.79125,0.16368,0.01387],\n                       [0.78260,0.15693,0.01264],\n                       [0.77377,0.15028,0.01148],\n                       [0.76476,0.14374,0.01041],\n                       [0.75556,0.13731,0.00942],\n                       [0.74617,0.13098,0.00851],\n                       [0.73661,0.12477,0.00769],\n                       [0.72686,0.11867,0.00695],\n                       [0.71692,0.11268,0.00629],\n                       [0.70680,0.10680,0.00571],\n                       [0.69650,0.10102,0.00522],\n                       [0.68602,0.09536,0.00481],\n                       [0.67535,0.08980,0.00449],\n                       [0.66449,0.08436,0.00424],\n                       [0.65345,0.07902,0.00408],\n                       [0.64223,0.07380,0.00401],\n                       [0.63082,0.06868,0.00401],\n                       [0.61923,0.06367,0.00410],\n                       [0.60746,0.05878,0.00427],\n                       [0.59550,0.05399,0.00453],\n                       [0.58336,0.04931,0.00486],\n                       [0.57103,0.04474,0.00529],\n                       [0.55852,0.04028,0.00579],\n                       [0.54583,0.03593,0.00638],\n                       [0.53295,0.03169,0.00705],\n                       [0.51989,0.02756,0.00780],\n                       [0.50664,0.02354,0.00863],\n                       [0.49321,0.01963,0.00955],\n                       [0.47960,0.01583,0.01055]])\n\n\n\n\ndef RGBToPyCmap(rgbdata):\n    nsteps = rgbdata.shape[0]\n    stepaxis = np.linspace(0, 1, nsteps)\n\n    rdata=[]; gdata=[]; bdata=[]\n    for istep in range(nsteps):\n        r = rgbdata[istep,0]\n        g = rgbdata[istep,1]\n        b = rgbdata[istep,2]\n        rdata.append((stepaxis[istep], r, r))\n        gdata.append((stepaxis[istep], g, g))\n        bdata.append((stepaxis[istep], b, b))\n\n    mpl_data = {'red':   rdata,\n                 'green': gdata,\n                 'blue':  bdata}\n\n    return mpl_data\n\n\nmpl_data = RGBToPyCmap(turbo_colormap_data)\nplt.register_cmap(name='turbo', data=mpl_data, lut=turbo_colormap_data.shape[0])\n\nmpl_data_r = RGBToPyCmap(turbo_colormap_data[::-1,:])\nplt.register_cmap(name='turbo_r', data=mpl_data_r, lut=turbo_colormap_data.shape[0])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np, pandas as pd, pydicom as dcm\nimport keras\nimport tensorflow as tf\nimport matplotlib.pyplot as plt, seaborn as sns\nimport os, glob\nimport bokeh\n\ntr = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/train.csv\")\nte = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/test.csv\")\nTRAIN_PATH = \"../input/rsna-str-pulmonary-embolism-detection/train/\"\nfiles = glob.glob('../input/rsna-str-pulmonary-embolism-detection/train/*/*/*.dcm')\ndef dicom_to_image(filename):\n    im = dcm.dcmread(filename)\n    img = im.pixel_array\n    img[img == -2000] = 0\n    return img\nsns.set()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Starting the EDA"},{"metadata":{},"cell_type":"markdown","source":"First of all, let's do a small preliminary check to our data. Take a look at how many scans we have per folder (how many CT slices we have per patient, in essence). Kaggle says this dataset is almost 1 TB, so training would be quite the conundrum on this full dataset."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print('Total patients {}'.format(len(os.listdir(TRAIN_PATH))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So it seems like we have an abundance of patients and an (over?)abundance of training data that we can utilize to feed our EfficientNets. Let's take a look at one of our DICOM files, just to take a brief look."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.axis('off')\nplt.imshow(-dcm.dcmread(\"../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/00ac73cfc372.dcm\").pixel_array, cmap='bone');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that the colors in the image above have been inverted by me for maximum visibility. This data format seems close to the images in the Data Science Bowl 2017 and the recent OSIC competition, and not too close to other lung-based competitions (the SIIM-ACR Pneumothorax Competition used larger images and was a segmentation task)."},{"metadata":{},"cell_type":"markdown","source":"# Meta-exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now this competition is a little (or a lot depending on your point of view) odd because of the submission scoring and the weighted log loss. In essence, our submission.csv file should have a number of rows equal to $n_{img} + (n_{studies} * n_{labels})$ where our $n$ basically serves as an indicator of \"number of\" and our metric is weighted on the basis of there being more important labels than others. They are giving the most importance with their weighted metric to \"Central PE\". Now what is this central PE you might ask?\n"},{"metadata":{},"cell_type":"markdown","source":"These are the principal data fields given by Kaggle on the competition's data page:\n```\n+ StudyInstanceUID - unique ID for each study (exam) in the data.\n+ SeriesInstanceUID - unique ID for each series within the study.\n+ SOPInstanceUID - unique ID for each image within the study (and data).\n+ pe_present_on_image - image-level, notes whether any form of PE is present on the image.\n+ negative_exam_for_pe - exam-level, whether there are any images in the study that have PE present.\n+ qa_motion - informational, indicates whether radiologists noted an issue with motion in the study.\n+ qa_contrast - informational, indicates whether radiologists noted an issue with contrast in the study.\n+ flow_artifact - informational\n+ rv_lv_ratio_gte_1 - exam-level, indicates whether the RV/LV ratio present in the study is >= 1\n+ rv_lv_ratio_lt_1 - exam-level, indicates whether the RV/LV ratio present in the study is < 1\n+ leftsided_pe - exam-level, indicates that there is PE present on the left side of the images in the study\n+ chronic_pe - exam-level, indicates that the PE in the study is chronic\n+ true_filling_defect_not_pe - informational, indicates a defect that is NOT PE\n+ rightsided_pe - exam-level, indicates that there is PE present on the right side of the images in the study\n+ acute_and_chronic_pe - exam-level, indicates that the PE present in the study is both acute AND chronic\n+ central_pe - exam-level, indicates that there is PE present in the center of the images in the study\n+ indeterminate -exam-level, indicates that while the study is not negative for PE, an ultimate set of exam-level labels could not be created, due to QA issues\n```"},{"metadata":{},"cell_type":"markdown","source":"Now, with the following data, it's possible to see that there are defects in a lung that are not a pulmonary embolism and are perhaps the effects of another pulmonary disease. We can explore the target columns in the metadata."},{"metadata":{"trusted":true},"cell_type":"code","source":"cat = []\nfor i in tr.columns: \n    if tr[i].dtype == int:\n        cat.append(i)\nprint(f\"{len(cat)} categorical targets are present in the data.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check the percentage of non-zero values, thanks to Konstantyin Isaienkov."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"zero_count_list = []\none_count_list = []\ncols_list = cat\nfor col in cols_list:\n    zero_count_list.append((tr[col]==0).sum())\n    one_count_list.append((tr[col]==1).sum())\n\nN = len(cols_list)\nind = np.arange(N)\nwidth = 0.35\n\nplt.figure(figsize=(6,10))\np1 = plt.barh(ind, zero_count_list, width, color='red')\np2 = plt.barh(ind, one_count_list, width, left=zero_count_list, color=\"blue\")\nplt.yticks(ind, cols_list)\nplt.legend((p1[0], p2[0]), ('Zero count', 'One Count'))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So from this, we gather that we have heavily imbalanced classes in the data and it will be difficult to work with these, like in SIIM-ISIC Melanoma Competition."},{"metadata":{},"cell_type":"markdown","source":"# Image-based analysis"},{"metadata":{},"cell_type":"markdown","source":"Time to get what all those who came here were looking for - the images."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"f, plots = plt.subplots(4, 5, sharex='col', sharey='row', figsize=(10, 8))\nfor i in range(20):\n    plots[i // 5, i % 5].axis('off')\n    plots[i // 5, i % 5].imshow(dicom_to_image(np.random.choice(files[0:20]), ),cmap='turbo') # first twenty images","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The turbo colormap data might look disgusting visually, but it helps you to get the contrasts in your image just right, to the point where any small difference in the image values gets exacerbated.\n\nThe images all look roughly similar to the Data Science Bowl and to the OSIC competition's images, which is helpful as we can apply existing preprocessing techniques(see \"Full Preprocessing Tutorial\" and \"Pulmonary Fibrosis Preprocessing\".) We also have the metadata fields, which could potentially be **very interesting** to examine:"},{"metadata":{"trusted":true},"cell_type":"code","source":"FNAME = dcm.dcmread(\"../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/00ac73cfc372.dcm\")\nprint(FNAME.file_meta)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So the class of this is a CT scan, the implementation is dcm4che (potentially expanded form DCM for Chest), and you have all the standard metadata of a DICOM file for the lungs. Do some little checks on the pixel array:"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"{FNAME.pixel_array.shape} is the size of the images provided.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So it seems we have two-dimensional 512x512 images provided to us. Let's check the raw image values:"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,1,figsize=(20,10))\nfor file in files[0:10]:\n    dataset = dcm.dcmread(file)\n    image = dataset.pixel_array.flatten()\n    rescaled_image = image * dataset.RescaleSlope + dataset.RescaleIntercept\n    sns.distplot(image.flatten(), ax=ax[0]);\n    sns.distplot(rescaled_image.flatten(), ax=ax[1])\nax[0].set_title(\"Raw pixel array distributions for 10 examples\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now it seems like out images mostly have a very odd distribution in the first 10, which will be interesting to consider in our future models. It's not exactly going to be ideal distribution for our future models but it will do. We also can perform some more checks on this data and the metadata, but let's move on to the idea of windowing. Credits to Jeremy Howard https://www.kaggle.com/jhoward/don-t-see-like-a-radiologist-fastai and David Tang https://www.kaggle.com/dcstang/see-like-a-radiologist-with-systematic-windowing"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(2,1, sharex='col', figsize=(9,9), gridspec_kw={'hspace': 0.1, 'wspace': 0})\ndef window_image(img, window_center,window_width, intercept, slope, rescale=True):\n\n    img = (img*slope +intercept)\n    img_min = window_center - window_width//2\n    img_max = window_center + window_width//2\n    img[img<img_min] = img_min\n    img[img>img_max] = img_max\n    \n    if rescale:\n        # Extra rescaling to 0-1, not in the original notebook\n        img = (img - img_min) / (img_max - img_min)\n    \n    return img\n\ndef get_first_of_dicom_field_as_int(x):\n    #get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)\n    if type(x) == dcm.multival.MultiValue:\n        return int(x[0])\n    else:\n        return int(x)\n\ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, #window center\n                    data[('0028','1051')].value, #window width\n                    data[('0028','1052')].value, #intercept\n                    data[('0028','1053')].value] #slope\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]\n\nwindow_center , window_width, intercept, slope = get_windowing(FNAME)\nfor ax in fig.axes:\n    ax.axis(\"off\")\nax1.imshow(window_image(FNAME.pixel_array, window_center , window_width, intercept, slope), cmap='Spectral');\nax2.imshow(FNAME.pixel_array,cmap='Spectral');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are also a lot more of things and ideas I have for this data, such as applying some of the work from OSIC and related competitions to this among others. Firstly, we'll need to do some more thorough works with EDA and secondly we need to try to train **more efficiently**. And windowing gives us meager humans a better insight, so **why not try?**"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(2,1, sharex='col', figsize=(9,9), gridspec_kw={'hspace': 0.1, 'wspace': 0})\ndef window_image(img, window_center,window_width, intercept, slope, rescale=True):\n\n    img = (img*slope +intercept)\n    img_min = window_center - window_width//2\n    img_max = window_center + window_width//2\n    img[img<img_min] = img_min\n    img[img>img_max] = img_max\n    \n    if rescale:\n        # Extra rescaling to 0-1, not in the original notebook\n        img = (img - img_min) / (img_max - img_min)\n    \n    return img\n\ndef get_first_of_dicom_field_as_int(x):\n    #get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)\n    if type(x) == dcm.multival.MultiValue:\n        return int(x[0])\n    else:\n        return int(x)\n\ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, #window center\n                    data[('0028','1051')].value, #window width\n                    data[('0028','1052')].value, #intercept\n                    data[('0028','1053')].value] #slope\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]\nFNAME = dcm.dcmread(\"../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/03d7693b0405.dcm\")\nwindow_center , window_width, intercept, slope = get_windowing(FNAME)\nfor ax in fig.axes:\n    ax.axis(\"off\")\nax1.imshow(window_image(FNAME.pixel_array, window_center , window_width, intercept, slope), cmap='Spectral');\nax2.imshow(FNAME.pixel_array,cmap='Spectral');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}