{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Index\n- [EDA](#EDA)\n- [Map visualisation](#wktmap)\n- [Metrics previous run](#metrics1)\n- [Training](#Training)\n- [Metrics](#metrics2)\n\n## Links to others parts\n- [Utility script: contrails_utils](https://www.kaggle.com/code/waechter/contrails-utils)\n- [Submission Notebook](https://www.kaggle.com/code/waechter/identify-contrail-submit-stack)\n- [Github](https://github.com/DomWch/kaggle-competitions)\n    - [Train history](https://www.kaggle.com/code/waechter/identifify-contrails-eda-pseudolabel-train) \n    - [First try: Custom Unet with tensorflow](https://www.kaggle.com/code/waechter/identify-contrails)\n    - [Full map visualisation](https://www.kaggle.com/code/waechter/wkt-map)\n\n\n## Thanks everyone 🚀 ! I learned a lot in this competition reading discussion and notebook\nIn particular:  \n- [egortrushin for [GR-ICRGW] Training with 4 folds](https://www.kaggle.com/code/egortrushin/gr-icrgw-training-with-4-folds/notebook)\n- [janmpia for his fast loading trick](https://www.kaggle.com/competitions/google-research-identify-contrails-reduce-global-warming/discussion/414549)\n- [One month to go! Summary of everything that happened](https://www.kaggle.com/competitions/google-research-identify-contrails-reduce-global-warming/discussion/420629)\n","metadata":{}},{"cell_type":"code","source":"%%capture\nimport contrails_utils as cu","metadata":{"_uuid":"168e4e1d-a044-4689-bed0-a3386f1b570d","_cell_guid":"03336978-6fc2-4f58-9a61-d09b9260aafd","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cu.Commun.colors.GREEN.format(\"contrails_utils imported\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### to reload a lib updated (if modified)\n\n# import importlib\n# importlib.reload(cu)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nimport os\nfrom typing import Literal\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nimport seaborn as sns\n\nfrom pathlib import Path","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Kaggle environment variable\nINTERACTIVE = os.getenv(\"KAGGLE_KERNEL_RUN_TYPE\") == \"Interactive\"\nRERUN = os.getenv(\"KAGGLE_IS_COMPETITION_RERUN\") is not None\n\n## Need accelarator to train\n## train on pseudo label take a long time since there are more data\nDO_PSEUDO_TRAIN, DO_TRAIN=(False, False)\nprint(f\"Interactive {INTERACTIVE}, RERUN {RERUN}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"data_path = Path(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming\")\npseudo_label_path = Path(f\"/kaggle/input/pseudolabelling/pseudolabel/train/pseudolabel\")\nHAVE_PSEUDO_DATA = pseudo_label_path.exists()\nHAVE_PSEUDO_DATA","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if HAVE_PSEUDO_DATA:\n#     cu.Visualization.plot_label_sequence(1001436206602675989)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if HAVE_PSEUDO_DATA:\n     display(cu.Visualization.plot_anim(1001436206602675989))","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if HAVE_PSEUDO_DATA:\n     display(cu.Visualization.plot_anim(1004713532062164491))","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if HAVE_PSEUDO_DATA:\n    display(cu.Visualization.plot_anim(1002294268041839364))","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"wktmap\"></a>\n# WKT map Visualization","metadata":{}},{"cell_type":"code","source":"metadata_val = pd.read_json(data_path/\"validation_metadata.json\")\nmetadata_val","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's strange record_id from metadata are not in file, but exluding the last 5digit we can always find one","metadata":{}},{"cell_type":"code","source":"for record_id in metadata_val.record_id[0:10]:\n    found = (data_path/f\"validation/{str(record_id)}\").exists()\n    print(f'record_id: {record_id} {\"Found\" if found else \"Not Found\"}')\n    if not found:\n        print(\"but fount: \", \" \".join([path.name for path in (data_path/\"validation\").glob(f\"*{str(record_id)[0:-5]}*\")]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_val.projection_wkt.values[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_train = pd.read_json(data_path/\"train_metadata.json\")\nmetadata_train.projection_wkt.values[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Same in train","metadata":{}},{"cell_type":"code","source":"for record_id in metadata_train.record_id[0:10]:\n    found = (data_path/f\"train/{str(record_id)}\").exists()\n    print(f'record_id: {record_id} {\"Found\" if found else \"Not Found\"}')\n    if not found:\n        print(\"but fount: \", \" \".join([path.name for path in (data_path/\"train\").glob(f\"*{str(record_id)[0:-5]}*\")]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"merge both","metadata":{}},{"cell_type":"code","source":"metadata_train[\"from\"]=\"train\"\nmetadata_val[\"from\"]=\"validation\"\nmetadata = pd.concat([metadata_train,metadata_val])\ndel (metadata_train, metadata_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata['datetime'] = pd.to_datetime(metadata.timestamp, unit='s')\nmetadata['date'] = metadata.datetime.dt.date\nmetadata['date']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_args = dict(legend=True, stacked=True, grid=True, figsize=(30,20))\nfig, ax = plt.subplots()\nax = metadata[metadata[\"from\"]==\"validation\"].groupby('date')['record_id'].count().plot( color=\"red\", **plot_args)\nax = metadata[metadata[\"from\"]==\"train\"].groupby('date')['record_id'].count().plot(kind=\"line\", color=\"green\", **plot_args)\nax.legend([\"validation\", \"train\"])\nax.set_title(\"Record per day\")\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"projection_wkt are all the same exept for central meridan so we extract it","metadata":{}},{"cell_type":"code","source":"reg = r'PROJCS\\[\"unknown\",GEOGCS\\[\"unknown\",DATUM\\[\"WGS_1984\",SPHEROID\\[\"WGS 84\",6378137,298\\.257223563,AUTHORITY\\[\"EPSG\",\"7030\"\\]\\],AUTHORITY\\[\"EPSG\",\"6326\"\\]\\],PRIMEM\\[\"Greenwich\",0,AUTHORITY\\[\"EPSG\",\"8901\"\\]\\],UNIT\\[\"degree\",0\\.0174532925199433,AUTHORITY\\[\"EPSG\",\"9122\"\\]\\]\\],PROJECTION\\[\"Transverse_Mercator\"\\],PARAMETER\\[\"latitude_of_origin\",0\\],PARAMETER\\[\"central_meridian\",(-?\\d+)\\],PARAMETER\\[\"scale_factor\",0\\.9996\\],PARAMETER\\[\"false_easting\",500000\\],PARAMETER\\[\"false_northing\",0\\],UNIT\\[\"metre\",1,AUTHORITY\\[\"EPSG\",\"9001\"\\]\\],AXIS\\[\"Easting\",EAST\\],AXIS\\[\"Northing\",NORTH\\]\\]'\nmetadata[\"central_meridian\"] = metadata.projection_wkt.str.extract(reg).astype(int)\nmetadata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## check that regex match all samples\nassert 0 == sum(metadata.central_meridian.isna()) == sum(metadata.central_meridian.isna())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.describe(datetime_is_numeric=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(metadata.loc[metadata[\"from\"]==\"train\"].describe(datetime_is_numeric=True))\ndisplay(metadata.loc[metadata[\"from\"]==\"validation\"].describe(datetime_is_numeric=True))","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Search for duplicate","metadata":{}},{"cell_type":"code","source":"duplicate_subset=[\"central_meridian\",\"timestamp\"]\nprint(f'{duplicate_subset} : {sum(metadata.duplicated(subset=duplicate_subset))}')\nduplicate_subset+=[\"row_min\",\"row_size\",\"col_min\",\"col_size\"]\nprint(f'{duplicate_subset} : {sum(metadata.duplicated(subset=duplicate_subset))}')\nduplicate_subset+=[\"timestamp\"]\nprint(f'{duplicate_subset} : {sum(metadata.duplicated(subset=duplicate_subset))}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_duplicate(df, subset):\n    return df[df.duplicated(subset=subset,keep=False)].sort_values(by=subset)\n\nshow_duplicate(metadata, [\"central_meridian\",\"timestamp\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_duplicate = show_duplicate(metadata,[\"central_meridian\", \"row_min\",\"row_size\",\"col_min\",\"col_size\"])\nall_duplicate","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_duplicate[all_duplicate[\"from\"]==\"validation\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"few presume duplicate (same location)","metadata":{}},{"cell_type":"code","source":"%%time\nfor i, row in all_duplicate[0:5].iterrows():\n    print(f\"{row['from']} {row.record_id} {row.timestamp} {row.row_min} {row.row_size}\")\n    cu.Visualization.plot_anim(str(row.record_id), path=(data_path/\"train\") if row[\"from\"]==\"train\" else (data_path/\"validation\"))","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Map\nplot the things!","metadata":{}},{"cell_type":"code","source":"%%time\nfor central_meridian in metadata.central_meridian.unique():\n    print(f\"plot map for central_longitude {central_meridian}\")\n    cu.Visualization.plot_crs(metadata,central_meridian)\n    \nmetadata[[\"central_meridian\",\"from\"]].value_counts(normalize=True).plot.pie(autopct='%1.1f%%', title=\"Percentage by meridian\",figsize=(30,30))\nplt.show()","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### all regroup in the same projection\ncu.Visualization.plot_merge_crs(metadata)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_metrics(path):\n    for metrics_path in path.glob(\"**/metrics.csv\"):\n        metrics = pd.read_csv(metrics_path, index_col=\"epoch\").bfill().groupby(\"epoch\").agg(\"first\").drop(columns=\"step\")\n        g = sns.relplot(data=metrics.drop(columns=\"lr\"), kind=\"line\")\n        plt.title(f\"{metrics_path.parents[3].name} {metrics_path.parents[2].name[5:25]}\")\n        plt.gcf().set_size_inches(20, 5)\n        metrics[\"lr_ratio\"] = metrics.lr/metrics.lr.max()\n        metrics.lr_ratio.plot()\n        plt.grid()\n        plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"metrics1\"></a>\n### Plot metrics from previous training\nand set up path to checkpoint","metadata":{}},{"cell_type":"code","source":"path_models = {\"unet384\":{\"path\":Path(\"/kaggle/input/unet384\"),\"image_size\":384},\n                \"from-scratch\":{\"path\":Path(\"/kaggle/input/from-scratch\"),\"image_size\":256,\"description\":\"smp unet imagenet default weight\"},\n               \"from-scratch386\":{\"path\":Path(\"/kaggle/input/from-scratch386\"),\"image_size\":386,\"description\":\"train on pseudolabel then train real data\"},\n              \"from-scratch512\":{\"path\":Path(\"/kaggle/input/from-scratch512\"),\"image_size\":512,\"description\":\"smp unet imagenet default weight\"},\n               \"from-scratch512-v2\":{\"path\":Path(\"/kaggle/input/gr-icrgw-training-with-4-folds\"),\"image_size\":384,\"description\":\"smp unet imagenet default weight\"},\n               \"4folds-original\":{\"path\":Path(\"/kaggle/input/gr-icrgw-training-with-4-folds\"),\"image_size\":384,\"description\":\"By egortrushin from https://www.kaggle.com/code/egortrushin/gr-icrgw-training-with-4-folds/notebook\"},\n               \"4folds-v2\":{\"path\":Path(\"/kaggle/input/training-with-4-folds\"),\"image_size\":384,\"description\":\"\"},\n               \"previous\":{\"path\":Path(\"/kaggle/input/pseudolabel-train/\"),\"description\":\"previous run saved model, read config from .yaml\"},\n              }\nfor name, val in path_models.items():\n    if val[\"path\"].exists():\n        print(f\"Plot metrics from {name} folder\")\n        plot_metrics(val[\"path\"])\n\n        \npath_previous_model = path_models[\"unet384\"][\"path\"]\n# plot_metrics(path_previous_model)\ncheckpoints = list(path_previous_model.glob(\"**/*.ckpt\"))\nif len(checkpoints)>0:\n    # take model with best score in this folder (path end with valdice=0.xxxx)\n    checkpoints.sort(key=lambda pa: int(pa.stem[-4]),reverse=True)\n    model_checkpoint_path = checkpoints[0]\n    print(f\"load weight checkpoint: {model_checkpoint_path}\")\nelse:\n    model_checkpoint_path=Path(\"from_scratch\")\n    print(\"from scratch start\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"config = cu.PyLModel.CFG.default_train_config() # open_config \nepochs = (1,2) if INTERACTIVE else (3,5)\nconfig","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_TIMES_BEFORE = 4\ndf_val = pd.DataFrame([{'record_id': path.name, 'path':path, 'target':path/\"human_pixel_masks.npy\", \"sequence_num\":N_TIMES_BEFORE} for path in (data_path/\"validation\").iterdir()])\nif INTERACTIVE:\n        df_val = df_val[0:100]\ndf_val","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train on pseudo label","metadata":{}},{"cell_type":"code","source":"%%time\nif HAVE_PSEUDO_DATA and DO_PSEUDO_TRAIN:\n    sequence_nums = [0,1,2,3,5,6,7]\n    df_train = pd.DataFrame([{'record_id': path.name, 'path':path, 'target':pseudo_label_path/f\"{path.name}/sequence_{sequence_num}/label.npy\",'sequence_num':sequence_num} for path in Path(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\").iterdir() for sequence_num in sequence_nums if (pseudo_label_path/f\"{path.name}/sequence_{sequence_num}\").exists()])\n    if INTERACTIVE:\n        df_train = df_train[0:100]\n        \n    config[\"trainer\"][\"min_epochs\"], config[\"trainer\"][\"max_epochs\"] = (1,1) if INTERACTIVE else (3,5)\n    model = cu.PyLModel.train_from_checkpoint(df_train, df_val, model_path=model_checkpoint_path, config=config)\n    model_checkpoint_path = Path(\"/kaggle/working\").glob(\"**/*.ckpt\").__next__()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train on real data","metadata":{}},{"cell_type":"code","source":"df_train = pd.DataFrame([{'record_id': path.name, 'path':path, 'target':path/\"human_pixel_masks.npy\", \"sequence_num\":N_TIMES_BEFORE} for path in (data_path/\"train\").iterdir()])\nif INTERACTIVE:\n    df_train = df_train[0:100]\ndf_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = contrails_utils.PyLModel.train_from_checkpoint(df, df_val, model_path=Path(\"/kaggle/input/pseudolabel-train/models/model-from-scratch-v2-real-train-val_dice=0.6047.ckpt\"), config=config)\nconfig[\"title\"]=\"Unet_real_train\"\nconfig[\"trainer\"][\"min_epochs\"], config[\"trainer\"][\"max_epochs\"] = epochs\nif DO_TRAIN:\n    if model_checkpoint_path.exists():\n        # set a very low lr for already trained model\n        config[\"model\"][\"optimizer_params\"] = {\"lr\":5e-7,\"weight_decay\": 0.01}\n    model = cu.PyLModel.train_from_checkpoint(df_train, df_val, model_path=model_checkpoint_path, config=config)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"metrics2\"></a>\n# metrics","metadata":{}},{"cell_type":"code","source":"plot_metrics(Path(\"/kaggle/working\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}