{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp -r ../input/modularized-own-code-det2/Detectron2_train/* ./\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-29T14:13:28.257236Z","iopub.execute_input":"2021-07-29T14:13:28.257573Z","iopub.status.idle":"2021-07-29T14:13:29.135831Z","shell.execute_reply.started":"2021-07-29T14:13:28.257499Z","shell.execute_reply":"2021-07-29T14:13:29.134795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:13:31.35836Z","iopub.execute_input":"2021-07-29T14:13:31.358715Z","iopub.status.idle":"2021-07-29T14:13:31.995087Z","shell.execute_reply.started":"2021-07-29T14:13:31.358683Z","shell.execute_reply":"2021-07-29T14:13:31.994192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat config/flags.py\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:13:38.338642Z","iopub.execute_input":"2021-07-29T14:13:38.339012Z","iopub.status.idle":"2021-07-29T14:13:38.980137Z","shell.execute_reply.started":"2021-07-29T14:13:38.338975Z","shell.execute_reply":"2021-07-29T14:13:38.979145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\ntorch.__version__","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:13:57.448811Z","iopub.execute_input":"2021-07-29T14:13:57.449182Z","iopub.status.idle":"2021-07-29T14:13:58.759901Z","shell.execute_reply.started":"2021-07-29T14:13:57.449147Z","shell.execute_reply":"2021-07-29T14:13:58.75912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvcc --version","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:14:01.627378Z","iopub.execute_input":"2021-07-29T14:14:01.6277Z","iopub.status.idle":"2021-07-29T14:14:02.270059Z","shell.execute_reply.started":"2021-07-29T14:14:01.627671Z","shell.execute_reply":"2021-07-29T14:14:02.269179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install detectron2 -f \\\n  https://dl.fbaipublicfiles.com/detectron2/wheels/cu110/torch1.7/index.html","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:18:15.663026Z","iopub.execute_input":"2021-07-29T14:18:15.663392Z","iopub.status.idle":"2021-07-29T14:18:49.498758Z","shell.execute_reply.started":"2021-07-29T14:18:15.663359Z","shell.execute_reply":"2021-07-29T14:18:49.497861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport torch\n\n# Our Detectron2_train Module imports \nfrom model.trainer import MyTrainer\nfrom utils.flags_util import save_flags, load_flags\nfrom utils.summarizer import vin_summarize\nfrom config.configurer import get_final_config\nfrom visualization.plot_metrics import plot_metrics\nfrom data.split_and_register import split_and_register\nfrom testing.generate_submissions import generate_submissions\nfrom visualization.visualize import visualize_submission_preds\n# from utils.kaggle import submit_to_kaggle\n\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\nfrom datetime import datetime as dt\nfrom datetime import timedelta as td\nnow = str((dt.now() + td(hours=5, minutes=30)).strftime(\"D_%Y-%m-%d_T_%H.%M.%S\"))   # capturing current time in India for logging experiments.\nnow = now[0:now.find(\":\")]\n\n\nfrom pycocotools.cocoeval import COCOeval\nprint(\"HACKING: overriding COCOeval.summarize = vin_summarize...\")\nCOCOeval.summarize = vin_summarize\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:19:19.054018Z","iopub.execute_input":"2021-07-29T14:19:19.054345Z","iopub.status.idle":"2021-07-29T14:19:19.06214Z","shell.execute_reply.started":"2021-07-29T14:19:19.054316Z","shell.execute_reply":"2021-07-29T14:19:19.061051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_kaggle_2CP = True\n\nflags_dict = {\n    \"debug\": False,\n    \"inputdir\": \"/kaggle/input\",\n    \"dataframe_dir\" : \"vinbigdata-dataframes-genpact-ds-distributions\",\n\n    \"outdir\": f\"v5_v1024_Kaggle_batch=4_{now}\", \n    \n    \"imgdir_name\": \"vinbigdata-chest-xray-resized-png-1024x1024\",\n    \"device\": \"cuda\",\n    \"split_mode\": \"val_minus_qa\",\n    \n    \"download_weights\": True,\n    \n    \"iter\": 30000,\n    \"ims_per_batch\": 4,\n    \"roi_batch_size_per_image\": 512,\n    \"eval_period\": 2000,\n    \"lr_scheduler_name\": \"WarmupCosineLR\",\n    \"base_lr\": 0.001,\n    \"num_workers\": 4,\n    \"aug_kwargs\": {\n        \"HorizontalFlip\": {\"p\": 0.5},\n        \"ShiftScaleRotate\": {\"scale_limit\": 0.15, \"rotate_limit\": 10, \"p\": 0.5},\n        \"RandomBrightnessContrast\": {\"p\": 0.5}\n    }\n}\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:29:50.792521Z","iopub.execute_input":"2021-07-29T14:29:50.792826Z","iopub.status.idle":"2021-07-29T14:29:50.799463Z","shell.execute_reply.started":"2021-07-29T14:29:50.792797Z","shell.execute_reply":"2021-07-29T14:29:50.798382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm dataset_dicts*\n!ls","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:19:43.067481Z","iopub.execute_input":"2021-07-29T14:19:43.067866Z","iopub.status.idle":"2021-07-29T14:19:44.325439Z","shell.execute_reply.started":"2021-07-29T14:19:43.067815Z","shell.execute_reply":"2021-07-29T14:19:44.324543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Output csv of 2 class classifier output\n# if use_kaggle_2CP:\n#     # Kaggle output from https://www.kaggle.com/awsaf49/vinbigdata-2class-prediction\n#     csv_2class_preds = \"/app/703268403/projects/Dicom/kaggle/ChestCT/vinbigdata-chest-xray-abnormalities-detection/Outputs/2_class/2-cls test pred.csv\"\n# else:\n#     csv_2class_preds = \"/app/703268403/projects/Dicom/kaggle/ChestCT/vinbigdata-chest-xray-abnormalities-detection/Outputs/detectron2/Sai/2_class/v1/test_pred_2class_Sai_7.7.21.csv\"\n\ncsv_sai_29_7 = \"../input/vinbigdata-dataframes-genpact-ds-distributions/test_pred_detect_SAI_29.7.21.csv\"\ncsv_kaggle_2cf = \"../input/vinbigdata-2class-prediction/2-cls test pred.csv\"\n   ","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:29:35.944258Z","iopub.execute_input":"2021-07-29T14:29:35.944629Z","iopub.status.idle":"2021-07-29T14:29:35.948827Z","shell.execute_reply.started":"2021-07-29T14:29:35.944596Z","shell.execute_reply":"2021-07-29T14:29:35.94781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Choice 1 : Use the flags in dict\nflags = save_flags(flags_dict)\n\n# # Choice 2 : Go to a previous output directory and get flags.\n# outdir = \"/app/703268403/projects/Dicom/kaggle/ChestCT/vinbigdata-chest-xray-abnormalities-detection/Outputs/detectron2/14_classes_v512/v4_D_2021-07-28_T_01.16.4\"\n# flags = load_flags(outdir)\n\nsplit_and_register(flags)\ncfg = get_final_config(flags)\n\ntrainer = MyTrainer(cfg)\ntrainer.resume_or_load(resume=False)\ntrainer.train()\n\noutdir = Path(cfg.OUTPUT_DIR)\nplot_metrics(outdir, flags.split_mode)\n\n# Prediction on Test set and kaggle submissions generation\n# 2 class filter on by default, can pass thresholds in parameters\nsub_csv_path_sai = generate_submissions(outdir , csv_sai_29_7, \n                                        pred_thr = 0.0, apply_2class_filter = True, \n                                        low_thr  = 0.08, high_thr = 0.95,\n                                        use_kaggle_2CP = False)\n\nsub_csv_path_K2cf = generate_submissions(outdir , csv_kaggle_2cf, \n                                        pred_thr = 0.0, apply_2class_filter = True, \n                                        low_thr  = 0.08, high_thr = 0.95,\n                                        use_kaggle_2CP = True)\n\nsub_csv_path_K2cf_04 = generate_submissions(outdir , csv_kaggle_2cf, \n                                        pred_thr = 0.0, apply_2class_filter = True, \n                                        low_thr  = 0.04, high_thr = 0.95,\n                                        use_kaggle_2CP = True)\n\nsub_csv_path_K2cf_04_01 = generate_submissions(outdir , csv_kaggle_2cf, \n                                        pred_thr = 0.01, apply_2class_filter = True, \n                                        low_thr  = 0.04, high_thr = 0.95,\n                                        use_kaggle_2CP = True)\n\n# submit_to_kaggle(sub_csv_path , \"WRITE SUBMISSION NOTES\")\n\n# Draw BBs over some samples from test set, from predictions just made.\n\ntest_dir = \"../input/vinbigdata-chest-xray-resized-png-1024x1024/test\"\n# THIS TIME : Visualize had hardcoded Test Directory : Deciding to do it on 45 later directly.\nvisualize_submission_preds(sub_csv_path_sai, test_dir, 20)\nvisualize_submission_preds(sub_csv_path_K2cf, test_dir, 20)\nvisualize_submission_preds(sub_csv_path_K2cf_04, test_dir, 20)\nvisualize_submission_preds(sub_csv_path_K2cf_04_01, test_dir, 20)","metadata":{"execution":{"iopub.status.busy":"2021-07-29T14:29:54.373263Z","iopub.execute_input":"2021-07-29T14:29:54.37358Z","iopub.status.idle":"2021-07-29T14:30:23.718002Z","shell.execute_reply.started":"2021-07-29T14:29:54.373551Z","shell.execute_reply":"2021-07-29T14:30:23.713736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a href=\"x.zip\"> Download Zipped File of my Outputs</a>","metadata":{"execution":{"iopub.status.busy":"2021-07-19T09:04:59.261958Z","iopub.execute_input":"2021-07-19T09:04:59.262321Z","iopub.status.idle":"2021-07-19T09:04:59.268941Z","shell.execute_reply.started":"2021-07-19T09:04:59.262265Z","shell.execute_reply":"2021-07-19T09:04:59.267446Z"}}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}