{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom fastai.data.all import *\nfrom fastai.vision.all import *\nimport torchvision.transforms as T\n","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:20.587619Z","iopub.execute_input":"2023-07-13T06:23:20.588115Z","iopub.status.idle":"2023-07-13T06:23:26.209193Z","shell.execute_reply.started":"2023-07-13T06:23:20.588071Z","shell.execute_reply":"2023-07-13T06:23:26.207823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = Path('/kaggle/input/google-research-identify-contrails-reduce-global-warming')\ntest_path = Path('/kaggle/input/google-research-identify-contrails-reduce-global-warming/test')\nvalid_path = Path('/kaggle/input/contrails-images-ash-color/contrails')","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.215407Z","iopub.execute_input":"2023-07-13T06:23:26.216094Z","iopub.status.idle":"2023-07-13T06:23:26.226354Z","shell.execute_reply.started":"2023-07-13T06:23:26.216051Z","shell.execute_reply":"2023-07-13T06:23:26.224906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_data_path = mkdir('test_data', overwrite=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.231365Z","iopub.execute_input":"2023-07-13T06:23:26.234521Z","iopub.status.idle":"2023-07-13T06:23:26.241271Z","shell.execute_reply.started":"2023-07-13T06:23:26.234479Z","shell.execute_reply":"2023-07-13T06:23:26.240135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run-Length Code\n\nThe following is code to both encode and decode RLE format.\n\nIMPORTANT: Unlike many previous Kaggle competition, empty predictions must be encoded as `'-'`. Empty string / null predictions will cause an error in scoring. The code below handles this change.","metadata":{}},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n    \"\"\"\n    Args:\n        x:  numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns: run length encoding as list\n    \"\"\"\n\n    dots = np.where(\n        x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\n\ndef list_to_string(x):\n    \"\"\"\n    Converts list to a string representation\n    Empty list returns '-'\n    \"\"\"\n    if x: # non-empty list\n        s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n    else:\n        s = '-'\n    return s\n\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n    '''\n    mask_rle: run-length as string formatted (start length)\n              empty predictions need to be encoded with '-'\n    shape: (height, width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    if mask_rle != '-': \n        s = mask_rle.split()\n        starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n        starts -= 1\n        ends = starts + lengths\n        for lo, hi in zip(starts, ends):\n            img[lo:hi] = 1\n    return img.reshape(shape, order='F')  # Needed to align to RLE direction","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.247255Z","iopub.execute_input":"2023-07-13T06:23:26.250012Z","iopub.status.idle":"2023-07-13T06:23:26.268309Z","shell.execute_reply.started":"2023-07-13T06:23:26.249972Z","shell.execute_reply":"2023-07-13T06:23:26.267252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{"execution":{"iopub.status.busy":"2023-07-06T03:00:46.474844Z","iopub.execute_input":"2023-07-06T03:00:46.475356Z","iopub.status.idle":"2023-07-06T03:00:46.483197Z","shell.execute_reply.started":"2023-07-06T03:00:46.475318Z","shell.execute_reply":"2023-07-06T03:00:46.481137Z"}}},{"cell_type":"code","source":"def read_record(record_id):\n    record_data = {}\n    for x in [\n        \"band_11\", \n        \"band_14\", \n        \"band_15\", \n    ]:\n\n        record_data[x] = np.load(os.path.join(record_id, x + \".npy\"))\n    \n    return record_data","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.273389Z","iopub.execute_input":"2023-07-13T06:23:26.27632Z","iopub.status.idle":"2023-07-13T06:23:26.285132Z","shell.execute_reply.started":"2023-07-13T06:23:26.276279Z","shell.execute_reply":"2023-07-13T06:23:26.28325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\nN_TIMES_BEFORE = 4","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.286707Z","iopub.execute_input":"2023-07-13T06:23:26.287889Z","iopub.status.idle":"2023-07-13T06:23:26.299284Z","shell.execute_reply.started":"2023-07-13T06:23:26.287847Z","shell.execute_reply":"2023-07-13T06:23:26.298227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_false_color(record_data):\n    _T11_BOUNDS = (243, 303)\n    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n    _TDIFF_BOUNDS = (-4, 2)\n\n    r = normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n    g = normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    img = false_color[..., N_TIMES_BEFORE]\n    \n    return img\n","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.302361Z","iopub.execute_input":"2023-07-13T06:23:26.305117Z","iopub.status.idle":"2023-07-13T06:23:26.317561Z","shell.execute_reply.started":"2023-07-13T06:23:26.305074Z","shell.execute_reply":"2023-07-13T06:23:26.31651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for record_i in test_path.ls():\n    data = read_record(record_i)\n    img = get_false_color(data)\n    final = np.dstack([img, np.zeros((256,256,1))])\n    final = final.astype(np.float16)\n    np.save(final_data_path/os.path.basename(record_i),final)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.322248Z","iopub.execute_input":"2023-07-13T06:23:26.323958Z","iopub.status.idle":"2023-07-13T06:23:26.510497Z","shell.execute_reply.started":"2023-07-13T06:23:26.323908Z","shell.execute_reply":"2023-07-13T06:23:26.509313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference on Validation Data","metadata":{}},{"cell_type":"code","source":"codes = ['Background', 'Contrail']","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.516161Z","iopub.execute_input":"2023-07-13T06:23:26.518802Z","iopub.status.idle":"2023-07-13T06:23:26.530439Z","shell.execute_reply.started":"2023-07-13T06:23:26.51874Z","shell.execute_reply":"2023-07-13T06:23:26.529284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_x(o):\n    return PILImage.create(T.ToPILImage()(T.ToTensor()(np.load(o)[...,:-1])))","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.539204Z","iopub.execute_input":"2023-07-13T06:23:26.541736Z","iopub.status.idle":"2023-07-13T06:23:26.549369Z","shell.execute_reply.started":"2023-07-13T06:23:26.541694Z","shell.execute_reply":"2023-07-13T06:23:26.548248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_y(o):\n    return PILMask.create(np.load(o)[...,-1].astype('int8'))","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.554856Z","iopub.execute_input":"2023-07-13T06:23:26.557645Z","iopub.status.idle":"2023-07-13T06:23:26.564947Z","shell.execute_reply.started":"2023-07-13T06:23:26.557604Z","shell.execute_reply":"2023-07-13T06:23:26.563838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_idx = list(range(1856))\nsplitter = IndexSplitter(valid_idx=valid_idx)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.569043Z","iopub.execute_input":"2023-07-13T06:23:26.57061Z","iopub.status.idle":"2023-07-13T06:23:26.580251Z","shell.execute_reply.started":"2023-07-13T06:23:26.570568Z","shell.execute_reply":"2023-07-13T06:23:26.57903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dblock= DataBlock(\n        blocks  = (ImageBlock, MaskBlock(codes=codes)),\n        get_x = get_x,\n        get_y = get_y,\n        splitter = splitter,\n        batch_tfms = [Normalize, Dihedral,],\n        )","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.582085Z","iopub.execute_input":"2023-07-13T06:23:26.582852Z","iopub.status.idle":"2023-07-13T06:23:26.599419Z","shell.execute_reply.started":"2023-07-13T06:23:26.582812Z","shell.execute_reply":"2023-07-13T06:23:26.598053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fnames = get_files(valid_path)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:26.602228Z","iopub.execute_input":"2023-07-13T06:23:26.6037Z","iopub.status.idle":"2023-07-13T06:23:33.730283Z","shell.execute_reply.started":"2023-07-13T06:23:26.603659Z","shell.execute_reply":"2023-07-13T06:23:33.729046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = dblock.dataloaders(fnames, bs=16)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:33.735965Z","iopub.execute_input":"2023-07-13T06:23:33.738615Z","iopub.status.idle":"2023-07-13T06:23:39.720957Z","shell.execute_reply.started":"2023-07-13T06:23:33.738572Z","shell.execute_reply":"2023-07-13T06:23:39.719792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = unet_learner(dls,\n                     resnet34,\n                     pretrained=False,                     \n                     loss_func=FocalLossFlat(axis=1),\n                     metrics=[Dice, foreground_acc],\n                     cbs=[CSVLogger(fname='t-dih.csv', append=True), \n                          SaveModelCallback(monitor='dice')],\n                     )","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:39.726742Z","iopub.execute_input":"2023-07-13T06:23:39.729432Z","iopub.status.idle":"2023-07-13T06:23:42.508898Z","shell.execute_reply.started":"2023-07-13T06:23:39.729388Z","shell.execute_reply":"2023-07-13T06:23:42.50771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('/kaggle/input/train-fastai-baseline/models/t')","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:42.510809Z","iopub.execute_input":"2023-07-13T06:23:42.511571Z","iopub.status.idle":"2023-07-13T06:23:47.043412Z","shell.execute_reply.started":"2023-07-13T06:23:42.511529Z","shell.execute_reply":"2023-07-13T06:23:47.042201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn.validate()\n# learn.show_results()","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:47.050498Z","iopub.execute_input":"2023-07-13T06:23:47.050978Z","iopub.status.idle":"2023-07-13T06:23:47.05949Z","shell.execute_reply.started":"2023-07-13T06:23:47.050936Z","shell.execute_reply":"2023-07-13T06:23:47.058338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# fastai Interpretation","metadata":{}},{"cell_type":"code","source":"interp = SegmentationInterpretation.from_learner(learn)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:23:47.064157Z","iopub.execute_input":"2023-07-13T06:23:47.064541Z","iopub.status.idle":"2023-07-13T06:24:19.109714Z","shell.execute_reply.started":"2023-07-13T06:23:47.064502Z","shell.execute_reply":"2023-07-13T06:24:19.108466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp.top_losses(k=5)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:24:19.115028Z","iopub.execute_input":"2023-07-13T06:24:19.117626Z","iopub.status.idle":"2023-07-13T06:24:19.144896Z","shell.execute_reply.started":"2023-07-13T06:24:19.117576Z","shell.execute_reply":"2023-07-13T06:24:19.143839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interp.show_results(interp.top_losses(k=5)[1])","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:24:19.149574Z","iopub.execute_input":"2023-07-13T06:24:19.152141Z","iopub.status.idle":"2023-07-13T06:24:20.930353Z","shell.execute_reply.started":"2023-07-13T06:24:19.152096Z","shell.execute_reply":"2023-07-13T06:24:20.922865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"execution":{"iopub.status.busy":"2023-07-06T10:12:22.256586Z","iopub.execute_input":"2023-07-06T10:12:22.257129Z","iopub.status.idle":"2023-07-06T10:12:22.262692Z","shell.execute_reply.started":"2023-07-06T10:12:22.257083Z","shell.execute_reply":"2023-07-06T10:12:22.261566Z"}}},{"cell_type":"code","source":"fnames = get_files(final_data_path)\n\nsubmission = pd.read_csv(data_path / 'sample_submission.csv', index_col='record_id')\nsubmission = submission[:0]\nfor rec in fnames:\n    pred,_ ,_ = learn.predict(rec)\n    # notice the we're converting rec to an `int` here:\n    submission.loc[int(rec.stem), 'encoded_pixels'] = list_to_string(rle_encode(pred))","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:24:20.933518Z","iopub.execute_input":"2023-07-13T06:24:20.934058Z","iopub.status.idle":"2023-07-13T06:24:21.457851Z","shell.execute_reply.started":"2023-07-13T06:24:20.934006Z","shell.execute_reply":"2023-07-13T06:24:21.449431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-13T06:24:21.459196Z","iopub.execute_input":"2023-07-13T06:24:21.459579Z","iopub.status.idle":"2023-07-13T06:24:21.487322Z","shell.execute_reply.started":"2023-07-13T06:24:21.45954Z","shell.execute_reply":"2023-07-13T06:24:21.470163Z"},"trusted":true},"execution_count":null,"outputs":[]}]}