{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-19T11:16:11.280298Z","iopub.execute_input":"2022-12-19T11:16:11.280691Z","iopub.status.idle":"2022-12-19T11:16:11.286026Z","shell.execute_reply.started":"2022-12-19T11:16:11.280629Z","shell.execute_reply":"2022-12-19T11:16:11.284871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport tensorflow as tf\nprint(tf.__version__)\nfrom pydantic import BaseModel\nfrom sklearn.model_selection import train_test_split","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-19T11:16:20.364855Z","iopub.execute_input":"2022-12-19T11:16:20.365867Z","iopub.status.idle":"2022-12-19T11:16:20.372883Z","shell.execute_reply.started":"2022-12-19T11:16:20.365804Z","shell.execute_reply":"2022-12-19T11:16:20.371378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"TPU in use!\")\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Strategy:\", strategy)\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-19T11:16:23.801382Z","iopub.execute_input":"2022-12-19T11:16:23.801774Z","iopub.status.idle":"2022-12-19T11:16:23.809374Z","shell.execute_reply.started":"2022-12-19T11:16:23.801738Z","shell.execute_reply":"2022-12-19T11:16:23.808152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config(BaseModel):\n    seed = 887\n    model_name = \"enetb1_v1\"\n    model_dir = \"/kaggle/input/g2net-tf-baseline/model/\"\n    # data\n \n    path_submission = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv\"\n\n    img_size = (360, 360)\n    channels = 3\n    img_shape = (*img_size, channels)\n    # model\n    base_model_weights = \"imagenet\"\n    dropout = 0.2\n    # training\n    shuffle_size = 128\n    epochs = 200\n    batch_size = 16 * strategy.num_replicas_in_sync\n    test_batch_size = 64\n    lr = 1e-4\n    patience = 15\n    \ncfg = Config()\ncfg.dict()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-19T11:16:24.526813Z","iopub.execute_input":"2022-12-19T11:16:24.527172Z","iopub.status.idle":"2022-12-19T11:16:24.538746Z","shell.execute_reply.started":"2022-12-19T11:16:24.52714Z","shell.execute_reply":"2022-12-19T11:16:24.537735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some things everyone knows:\n* Truth Seeker has asked:\n> Can someone please explain why data generation is needed for this competition? I've read through the description and all effort is on \"how\" data can be generated, not \"why\".\n\n>not quite right, there is a lot to learn: https://www.glowscript.org/#/user/grahamwoan/folder/Public/program/gwgw\n* Chris has a great i mean a great script for low noise signals:\nhttps://www.kaggle.com/code/chris62/generating-low-snr-gravity-waves/data\n\nthere are also some interesting experiments with low SNR\n* VLADIMIR SLAYKOVSKIY also has a great notebook G2NET: Generating Pure Signal\nhttps://www.kaggle.com/code/vslaykovsky/g2net-generating-pure-signal\nand also a good ideea:\n> Here I'm giving away a strategy that will likely give a huge advantage if implemented correctly. In short, it might be possible to reverse engineer the injected CW signal with Ligo&Virgo data.\n\nThere is a lot of effort to find the right training set for use with efficientnet model in various flavours\n\n\nSintetic data:\na lot of discussion here\nhttps://www.kaggle.com/competitions/g2net-detecting-continuous-gravitational-waves/discussion/361562\nmain pointfrom Rodrigo:\n> Still, I would be very grateful if you can tell me whether the F1 used in this competition is positive, or negative, or both.\n>Using Chris experiments : the - value is giving a boost for LB (0.3 AUC) but its is just an experiment An yes Chis is ( or was) using negative values.","metadata":{}},{"cell_type":"markdown","source":"Why the flipping coin model? \n\nVLADIMIR SLAYKOVSKIY proposal \"Reverse engineering 20% of test samples with external data\"\nBut i've tested a simple model with very very poor performance: 0.533 in LB\nSee the results of classification below, i'm not jumping to conclusions.\nBut maybe a two model approach is needed: one for gaussian noise and another one for real noise?\n\nAn in this case,in competition how can we disseminate when to apply one model or another :)","metadata":{}},{"cell_type":"code","source":"1+1","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:25.185091Z","iopub.execute_input":"2022-12-19T11:16:25.185443Z","iopub.status.idle":"2022-12-19T11:16:25.192747Z","shell.execute_reply.started":"2022-12-19T11:16:25.185412Z","shell.execute_reply":"2022-12-19T11:16:25.191528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(cfg.model_dir)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:25.392704Z","iopub.execute_input":"2022-12-19T11:16:25.393026Z","iopub.status.idle":"2022-12-19T11:16:25.405093Z","shell.execute_reply.started":"2022-12-19T11:16:25.392997Z","shell.execute_reply":"2022-12-19T11:16:25.40406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=pd.read_csv('/kaggle/input/360x360g2netpng/test_files.csv')\n#test=test.rename(columns={0: \"calea\", 1: \"id\"})\ntest = test.loc[:, ~test.columns.str.contains('^Unnamed')]\ntest=test.rename(columns={'0': \"calea\", '1': \"id\"}, errors=\"raise\")\n# test.calea=test.calea.str.replace('/kaggle/working/test/',\"/kaggle/input/360x360g2netpng/test/\")\ntest.calea=test.calea.str.replace('/kaggle/working/test/',\"/kaggle/input/360x360g2netpng/width128/test/\")\n# /kaggle/input/360x360g2netpng/width128/train/\ntest.calea=test.calea.str.replace('.jpg',\".png\")","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:25.588531Z","iopub.execute_input":"2022-12-19T11:16:25.588823Z","iopub.status.idle":"2022-12-19T11:16:25.625875Z","shell.execute_reply.started":"2022-12-19T11:16:25.588796Z","shell.execute_reply":"2022-12-19T11:16:25.624821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras_preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers, Model, Input, losses, metrics, optimizers, callbacks","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:25.823604Z","iopub.execute_input":"2022-12-19T11:16:25.823913Z","iopub.status.idle":"2022-12-19T11:16:25.829396Z","shell.execute_reply.started":"2022-12-19T11:16:25.823885Z","shell.execute_reply":"2022-12-19T11:16:25.828397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR_train = '/kaggle/input/360x360g2netpng/width360/train/'\n\nINPUT_DIR_test = '/kaggle/input/360x360g2netpng/width360/test/'","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:26.011259Z","iopub.execute_input":"2022-12-19T11:16:26.01191Z","iopub.status.idle":"2022-12-19T11:16:26.016825Z","shell.execute_reply.started":"2022-12-19T11:16:26.011872Z","shell.execute_reply":"2022-12-19T11:16:26.015805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idg = ImageDataGenerator()\ntest_generator  = idg.flow_from_dataframe(\n    test,\n    INPUT_DIR_test,\n    x_col = \"calea\",\n    y_col =None,\n    class_mode=None,\n    shuffle = False,\n    target_size=(360, 360),\n    batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:26.2042Z","iopub.execute_input":"2022-12-19T11:16:26.204471Z","iopub.status.idle":"2022-12-19T11:16:29.332951Z","shell.execute_reply.started":"2022-12-19T11:16:26.204446Z","shell.execute_reply":"2022-12-19T11:16:29.331223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded_model = tf.keras.models.load_model(cfg.model_dir)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:29.334788Z","iopub.execute_input":"2022-12-19T11:16:29.33507Z","iopub.status.idle":"2022-12-19T11:16:30.947462Z","shell.execute_reply.started":"2022-12-19T11:16:29.335042Z","shell.execute_reply":"2022-12-19T11:16:30.946486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Explore why a garbage score has two nice classified classes","metadata":{}},{"cell_type":"code","source":"preds = loaded_model.predict(test_generator, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:16:30.958251Z","iopub.execute_input":"2022-12-19T11:16:30.960603Z","iopub.status.idle":"2022-12-19T11:17:28.303936Z","shell.execute_reply.started":"2022-12-19T11:16:30.960556Z","shell.execute_reply":"2022-12-19T11:17:28.302898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del loaded_model","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.306858Z","iopub.execute_input":"2022-12-19T11:17:28.307512Z","iopub.status.idle":"2022-12-19T11:17:28.319458Z","shell.execute_reply.started":"2022-12-19T11:17:28.307469Z","shell.execute_reply":"2022-12-19T11:17:28.317985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.320482Z","iopub.execute_input":"2022-12-19T11:17:28.320835Z","iopub.status.idle":"2022-12-19T11:17:28.658597Z","shell.execute_reply.started":"2022-12-19T11:17:28.32078Z","shell.execute_reply":"2022-12-19T11:17:28.657457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['probability'] = preds","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.660178Z","iopub.execute_input":"2022-12-19T11:17:28.660639Z","iopub.status.idle":"2022-12-19T11:17:28.666883Z","shell.execute_reply.started":"2022-12-19T11:17:28.660602Z","shell.execute_reply":"2022-12-19T11:17:28.665854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR_submission='../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv'\nsample_submission=pd.read_csv(INPUT_DIR_submission)\nsample_submission.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.668251Z","iopub.execute_input":"2022-12-19T11:17:28.669325Z","iopub.status.idle":"2022-12-19T11:17:28.696529Z","shell.execute_reply.started":"2022-12-19T11:17:28.66929Z","shell.execute_reply":"2022-12-19T11:17:28.695707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['probability']=sample_submission['id'].map(test.set_index('id')['probability'])","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.699625Z","iopub.execute_input":"2022-12-19T11:17:28.699898Z","iopub.status.idle":"2022-12-19T11:17:28.711311Z","shell.execute_reply.started":"2022-12-19T11:17:28.699873Z","shell.execute_reply":"2022-12-19T11:17:28.710225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.714608Z","iopub.execute_input":"2022-12-19T11:17:28.714935Z","iopub.status.idle":"2022-12-19T11:17:28.727579Z","shell.execute_reply.started":"2022-12-19T11:17:28.714909Z","shell.execute_reply":"2022-12-19T11:17:28.726533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['target']=sample_submission['probability']","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.731925Z","iopub.execute_input":"2022-12-19T11:17:28.732175Z","iopub.status.idle":"2022-12-19T11:17:28.73694Z","shell.execute_reply.started":"2022-12-19T11:17:28.732151Z","shell.execute_reply":"2022-12-19T11:17:28.735993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" sample_submission.describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.738651Z","iopub.execute_input":"2022-12-19T11:17:28.739349Z","iopub.status.idle":"2022-12-19T11:17:28.763083Z","shell.execute_reply.started":"2022-12-19T11:17:28.739314Z","shell.execute_reply":"2022-12-19T11:17:28.762241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.765956Z","iopub.execute_input":"2022-12-19T11:17:28.766212Z","iopub.status.idle":"2022-12-19T11:17:28.909776Z","shell.execute_reply.started":"2022-12-19T11:17:28.766182Z","shell.execute_reply":"2022-12-19T11:17:28.908883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(data=sample_submission, x=\"target\")","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:28.911036Z","iopub.execute_input":"2022-12-19T11:17:28.913236Z","iopub.status.idle":"2022-12-19T11:17:29.325632Z","shell.execute_reply.started":"2022-12-19T11:17:28.913207Z","shell.execute_reply":"2022-12-19T11:17:29.324669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:29.327246Z","iopub.execute_input":"2022-12-19T11:17:29.327585Z","iopub.status.idle":"2022-12-19T11:17:29.353071Z","shell.execute_reply.started":"2022-12-19T11:17:29.32755Z","shell.execute_reply":"2022-12-19T11:17:29.35224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1+1","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:29.35452Z","iopub.execute_input":"2022-12-19T11:17:29.354894Z","iopub.status.idle":"2022-12-19T11:17:29.361022Z","shell.execute_reply.started":"2022-12-19T11:17:29.354859Z","shell.execute_reply":"2022-12-19T11:17:29.359944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:29.362607Z","iopub.execute_input":"2022-12-19T11:17:29.36307Z","iopub.status.idle":"2022-12-19T11:17:30.352203Z","shell.execute_reply.started":"2022-12-19T11:17:29.363034Z","shell.execute_reply":"2022-12-19T11:17:30.351033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['calea']=\"/kaggle/input/360x360g2netpng/width360/test/\"+sample_submission.id+\".jpg\"","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:30.355836Z","iopub.execute_input":"2022-12-19T11:17:30.356171Z","iopub.status.idle":"2022-12-19T11:17:30.367128Z","shell.execute_reply.started":"2022-12-19T11:17:30.356138Z","shell.execute_reply":"2022-12-19T11:17:30.366073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"low_score=sample_submission[sample_submission[\"probability\"]<=0.7]\nlow_score=low_score.reset_index()\nhigh_score=sample_submission[sample_submission[\"probability\"]>=0.9]\nhigh_score=high_score.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:30.368955Z","iopub.execute_input":"2022-12-19T11:17:30.369529Z","iopub.status.idle":"2022-12-19T11:17:30.380804Z","shell.execute_reply.started":"2022-12-19T11:17:30.369492Z","shell.execute_reply":"2022-12-19T11:17:30.379662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfont = {'family' : 'normal',\n        'weight' : 'bold',\n        'size'   : 7}\nplt.rc('font', **font)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:30.382382Z","iopub.execute_input":"2022-12-19T11:17:30.382761Z","iopub.status.idle":"2022-12-19T11:17:30.387953Z","shell.execute_reply.started":"2022-12-19T11:17:30.382727Z","shell.execute_reply":"2022-12-19T11:17:30.386722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\n# for images, labels in train_ds.take(1):\ni=0\nfor data, row in sample_submission[:9].iterrows():\n        #print(row)\n        ax = plt.subplot(3, 3, i + 1)\n        #print(row['id'])\n        image_path=row['calea']\n        image = plt.imread(image_path)\n        plt.imshow(image)\n#         plt.imshow(image_path)\n#         plt.title(labels[i].numpy())\n#         £print(labels[i].numpy())\n        plt.axis(\"off\")\n        i=i+1\n        #plt.savefig(filename)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:30.389784Z","iopub.execute_input":"2022-12-19T11:17:30.390181Z","iopub.status.idle":"2022-12-19T11:17:31.396531Z","shell.execute_reply.started":"2022-12-19T11:17:30.390145Z","shell.execute_reply":"2022-12-19T11:17:31.395674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 15))\ni=0\nfor data, row in high_score[:18].iterrows():\n        #print(row)\n        ax = plt.subplot(6, 3, i + 1)\n        #print(row['id'])\n        image_path=row['calea']\n        image = plt.imread(image_path)\n        plt.imshow(image)\n#         plt.imshow(image_path)\n#         plt.title(labels[i].numpy())\n#         £print(labels[i].numpy())\n        plt.axis(\"off\")\n        i=i+1\n        #plt.savefig(filename)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:31.397643Z","iopub.execute_input":"2022-12-19T11:17:31.398001Z","iopub.status.idle":"2022-12-19T11:17:33.171336Z","shell.execute_reply.started":"2022-12-19T11:17:31.397964Z","shell.execute_reply":"2022-12-19T11:17:33.170542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 15))\ni=0\nfor data, row in low_score[:18].iterrows():\n        #print(row)\n        ax = plt.subplot(6, 3, i + 1)\n        #print(row['id'])\n        image_path=row['calea']\n        image = plt.imread(image_path)\n        plt.imshow(image)\n#         plt.imshow(image_path)\n#         plt.title(labels[i].numpy())\n#         £print(labels[i].numpy())\n        plt.axis(\"off\")\n        i=i+1\n        #plt.savefig(filename)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:33.17283Z","iopub.execute_input":"2022-12-19T11:17:33.173572Z","iopub.status.idle":"2022-12-19T11:17:35.545538Z","shell.execute_reply.started":"2022-12-19T11:17:33.173536Z","shell.execute_reply":"2022-12-19T11:17:35.544728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test ","metadata":{}},{"cell_type":"code","source":"sample_submission_copy = sample_submission.copy()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:35.546841Z","iopub.execute_input":"2022-12-19T11:17:35.547336Z","iopub.status.idle":"2022-12-19T11:17:35.553558Z","shell.execute_reply.started":"2022-12-19T11:17:35.547304Z","shell.execute_reply":"2022-12-19T11:17:35.55276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = sample_submission.probability < 0.5\ncolumn_name = '\ttarget'\nsample_submission.loc[mask, column_name] = 0","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:35.554952Z","iopub.execute_input":"2022-12-19T11:17:35.555646Z","iopub.status.idle":"2022-12-19T11:17:35.567891Z","shell.execute_reply.started":"2022-12-19T11:17:35.555612Z","shell.execute_reply":"2022-12-19T11:17:35.566737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.loc[sample_submission.probability <0.5, 'target'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:35.569557Z","iopub.execute_input":"2022-12-19T11:17:35.570381Z","iopub.status.idle":"2022-12-19T11:17:35.574762Z","shell.execute_reply.started":"2022-12-19T11:17:35.570344Z","shell.execute_reply":"2022-12-19T11:17:35.573742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.loc[sample_submission.probability >0.95, 'target'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:35.57621Z","iopub.execute_input":"2022-12-19T11:17:35.576902Z","iopub.status.idle":"2022-12-19T11:17:35.584623Z","shell.execute_reply.started":"2022-12-19T11:17:35.576867Z","shell.execute_reply":"2022-12-19T11:17:35.583615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = sample_submission.probability > 0.95\ncolumn_name = '\ttarget'\nsample_submission.loc[mask, column_name] = 1","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:35.586126Z","iopub.execute_input":"2022-12-19T11:17:35.586838Z","iopub.status.idle":"2022-12-19T11:17:35.595154Z","shell.execute_reply.started":"2022-12-19T11:17:35.586803Z","shell.execute_reply":"2022-12-19T11:17:35.5943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(data=sample_submission, x=\"probability\")","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:35.602955Z","iopub.execute_input":"2022-12-19T11:17:35.603218Z","iopub.status.idle":"2022-12-19T11:17:36.030561Z","shell.execute_reply.started":"2022-12-19T11:17:35.603188Z","shell.execute_reply":"2022-12-19T11:17:36.029559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(data=sample_submission, x=\"target\")","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:36.03261Z","iopub.execute_input":"2022-12-19T11:17:36.033616Z","iopub.status.idle":"2022-12-19T11:17:36.434997Z","shell.execute_reply.started":"2022-12-19T11:17:36.033579Z","shell.execute_reply":"2022-12-19T11:17:36.434038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:36.436582Z","iopub.execute_input":"2022-12-19T11:17:36.436943Z","iopub.status.idle":"2022-12-19T11:17:36.442102Z","shell.execute_reply.started":"2022-12-19T11:17:36.436906Z","shell.execute_reply":"2022-12-19T11:17:36.441047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[['id','target']]=sample_submission[['id','target']]","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:36.44384Z","iopub.execute_input":"2022-12-19T11:17:36.444696Z","iopub.status.idle":"2022-12-19T11:17:36.454176Z","shell.execute_reply.started":"2022-12-19T11:17:36.444635Z","shell.execute_reply":"2022-12-19T11:17:36.45328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:36.455735Z","iopub.execute_input":"2022-12-19T11:17:36.45615Z","iopub.status.idle":"2022-12-19T11:17:36.471168Z","shell.execute_reply.started":"2022-12-19T11:17:36.456116Z","shell.execute_reply":"2022-12-19T11:17:36.469877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission_wave.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:36.473103Z","iopub.execute_input":"2022-12-19T11:17:36.473614Z","iopub.status.idle":"2022-12-19T11:17:36.496972Z","shell.execute_reply.started":"2022-12-19T11:17:36.473546Z","shell.execute_reply":"2022-12-19T11:17:36.496119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### proof of concept\nLoad models","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:36.498352Z","iopub.execute_input":"2022-12-19T11:17:36.498719Z","iopub.status.idle":"2022-12-19T11:17:47.94577Z","shell.execute_reply.started":"2022-12-19T11:17:36.498683Z","shell.execute_reply":"2022-12-19T11:17:47.944508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.float32),\n        'id'                   : tf.io.FixedLenFeature([], tf.string),\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float32)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    #print(example['spectrogram'].shape)\n    return example['spectrogram'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'id'                   : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float32)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    return example['spectrogram'], example['id'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=False):    \n    if augment:\n        img = augmentation(img)\n# for testing withou a dummy function should be created: \n                                            # def augmentation(img):\n                                            #     img = img\n                                            #     return img\n    #The tf.reshape does not change the order of or the total number of elements in the tensor, \n    #and so it can reuse the underlying data buffer. \n    #This makes it a fast operation independent of how big of a tensor it is operating on.\n    img = tf.reshape(img, [*IMG_SIZE, 2])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\ndef get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=32):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8 if tpu else 128)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    #Augumentation here\n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment,), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:47.948078Z","iopub.execute_input":"2022-12-19T11:17:47.948505Z","iopub.status.idle":"2022-12-19T11:17:47.966391Z","shell.execute_reply.started":"2022-12-19T11:17:47.948461Z","shell.execute_reply":"2022-12-19T11:17:47.965266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:47.969812Z","iopub.execute_input":"2022-12-19T11:17:47.97016Z","iopub.status.idle":"2022-12-19T11:17:48.16819Z","shell.execute_reply.started":"2022-12-19T11:17:47.970132Z","shell.execute_reply":"2022-12-19T11:17:48.167155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nGCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('tfrecg2netdataset2')\nfiles_test  = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:17:48.170542Z","iopub.execute_input":"2022-12-19T11:17:48.171533Z","iopub.status.idle":"2022-12-19T11:18:27.62986Z","shell.execute_reply.started":"2022-12-19T11:17:48.171493Z","shell.execute_reply":"2022-12-19T11:18:27.627626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=32","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.631722Z","iopub.execute_input":"2022-12-19T11:18:27.632472Z","iopub.status.idle":"2022-12-19T11:18:27.637454Z","shell.execute_reply.started":"2022-12-19T11:18:27.632431Z","shell.execute_reply":"2022-12-19T11:18:27.63647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        #policy = mixed_precision.Policy('mixed_bfloat16')\n        #mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE *= strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.638945Z","iopub.execute_input":"2022-12-19T11:18:27.639554Z","iopub.status.idle":"2022-12-19T11:18:27.651048Z","shell.execute_reply.started":"2022-12-19T11:18:27.639517Z","shell.execute_reply":"2022-12-19T11:18:27.649073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE=(360,360)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.653082Z","iopub.execute_input":"2022-12-19T11:18:27.653376Z","iopub.status.idle":"2022-12-19T11:18:27.662112Z","shell.execute_reply.started":"2022-12-19T11:18:27.653352Z","shell.execute_reply":"2022-12-19T11:18:27.660978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    inp = tf.keras.layers.Input(shape=(*IMG_SIZE, 2))\n    in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n    base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n    x = in_conv(inp)\n    x = base(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-5) \n    model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.663764Z","iopub.execute_input":"2022-12-19T11:18:27.664171Z","iopub.status.idle":"2022-12-19T11:18:27.673631Z","shell.execute_reply.started":"2022-12-19T11:18:27.664133Z","shell.execute_reply":"2022-12-19T11:18:27.67271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 5e-5\n    lr_max     = 5e-4\n    lr_min     = 1e-5\n    lr_ramp_ep = 4\n    lr_sus_ep  = 4\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.675195Z","iopub.execute_input":"2022-12-19T11:18:27.67561Z","iopub.status.idle":"2022-12-19T11:18:27.685435Z","shell.execute_reply.started":"2022-12-19T11:18:27.675574Z","shell.execute_reply":"2022-12-19T11:18:27.684478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # BUILD MODEL\n# K.clear_session()\n# with strategy.scope():\n#     model = build_model()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.686605Z","iopub.execute_input":"2022-12-19T11:18:27.687041Z","iopub.status.idle":"2022-12-19T11:18:27.698339Z","shell.execute_reply.started":"2022-12-19T11:18:27.686998Z","shell.execute_reply":"2022-12-19T11:18:27.697391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.699974Z","iopub.execute_input":"2022-12-19T11:18:27.700773Z","iopub.status.idle":"2022-12-19T11:18:27.707718Z","shell.execute_reply.started":"2022-12-19T11:18:27.700722Z","shell.execute_reply":"2022-12-19T11:18:27.706701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.load_weights('fold-%i.h5'%fold)\n# model.load_weights('/kaggle/input/g2net-tf-keras-train-test/fold-0.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.709201Z","iopub.execute_input":"2022-12-19T11:18:27.709865Z","iopub.status.idle":"2022-12-19T11:18:27.718544Z","shell.execute_reply.started":"2022-12-19T11:18:27.709831Z","shell.execute_reply":"2022-12-19T11:18:27.717704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # PREDICT OOF USING TTA\n# #     print('Predicting OOF ...')\n# TTA=1\n# ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n# ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n# pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] ","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.719965Z","iopub.execute_input":"2022-12-19T11:18:27.720872Z","iopub.status.idle":"2022-12-19T11:18:27.728496Z","shell.execute_reply.started":"2022-12-19T11:18:27.720832Z","shell.execute_reply":"2022-12-19T11:18:27.727566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission[['preds']]=pred","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.731617Z","iopub.execute_input":"2022-12-19T11:18:27.731907Z","iopub.status.idle":"2022-12-19T11:18:27.737292Z","shell.execute_reply.started":"2022-12-19T11:18:27.731883Z","shell.execute_reply":"2022-12-19T11:18:27.736366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # BUILD MODEL\n# K.clear_session()\n# with strategy.scope():\n#     model = build_model()\n# model.load_weights('/kaggle/input/g2net-tf-keras-train-test/fold-1.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.738818Z","iopub.execute_input":"2022-12-19T11:18:27.739304Z","iopub.status.idle":"2022-12-19T11:18:27.747164Z","shell.execute_reply.started":"2022-12-19T11:18:27.739269Z","shell.execute_reply":"2022-12-19T11:18:27.746289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # PREDICT OOF USING TTA\n# #     print('Predicting OOF ...')\n# TTA=1\n# ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n# ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n# pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] ","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.748764Z","iopub.execute_input":"2022-12-19T11:18:27.749149Z","iopub.status.idle":"2022-12-19T11:18:27.756417Z","shell.execute_reply.started":"2022-12-19T11:18:27.749094Z","shell.execute_reply":"2022-12-19T11:18:27.755538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2 if tpu else 1\nDISPLAY_PLOT = True","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.758216Z","iopub.execute_input":"2022-12-19T11:18:27.758647Z","iopub.status.idle":"2022-12-19T11:18:27.766043Z","shell.execute_reply.started":"2022-12-19T11:18:27.758612Z","shell.execute_reply":"2022-12-19T11:18:27.765074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# K.clear_session()\n# with strategy.scope():\n#     model = build_model()\n# files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\n# model.load_weights(f'/kaggle/input/low-g2net-tf-keras-train-test/fold-1.h5')\n# TTA=1\n# ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             shuffle=False,batch_size=BATCH_SIZE*4)\n# ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n# pred = model.predict(ds_test,verbose=VERBOSE)#[:TTA*ct_test,]\n# sample_submission[['preds1']]=pred\n# print(f'/kaggle/input/g2net-tf-keras-train-test/fold-1.h5')\n\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.767689Z","iopub.execute_input":"2022-12-19T11:18:27.76817Z","iopub.status.idle":"2022-12-19T11:18:27.775596Z","shell.execute_reply.started":"2022-12-19T11:18:27.768137Z","shell.execute_reply":"2022-12-19T11:18:27.774669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect(generation=2) \ngc.collect(generation=1) \ngc.collect(generation=0) ","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:27.777185Z","iopub.execute_input":"2022-12-19T11:18:27.777576Z","iopub.status.idle":"2022-12-19T11:18:28.049845Z","shell.execute_reply.started":"2022-12-19T11:18:27.777541Z","shell.execute_reply":"2022-12-19T11:18:28.048748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:28.051681Z","iopub.execute_input":"2022-12-19T11:18:28.052274Z","iopub.status.idle":"2022-12-19T11:18:28.067225Z","shell.execute_reply.started":"2022-12-19T11:18:28.052208Z","shell.execute_reply":"2022-12-19T11:18:28.0662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# K.clear_session()\n# files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\n# with strategy.scope():\n#     model = build_model()\n\n# model.load_weights(f'/kaggle/input/low-g2net-tf-keras-train-test/fold-1.h5')\n# TTA=1\n# ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,batch_size=BATCH_SIZE)\n# #ct_test = count_data_items(files_test);\n# pred = model.predict(ds_test,verbose=VERBOSE)#[:TTA*ct_test,]\n# sample_submission[[f'preds1']]=pred\n# print(f'/kaggle/input/g2net-tf-keras-train-test/fold-1.h5')\n# var=var+1\n\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:28.068836Z","iopub.execute_input":"2022-12-19T11:18:28.069195Z","iopub.status.idle":"2022-12-19T11:18:28.074303Z","shell.execute_reply.started":"2022-12-19T11:18:28.069159Z","shell.execute_reply":"2022-12-19T11:18:28.073234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BUILD MODEL\nvar=0\nTTA=1\nfiles_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\nds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*2)\nct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/2\nfor i in range(2):\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n    \n    model.load_weights(f'/kaggle/input/g2net-tf-keras-train-test/fold-{var}.h5'.format(var))\n\n#     ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#                 repeat=True,shuffle=False,batch_size=BATCH_SIZE*2)\n#     ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/2\n#     ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,batch_size=BATCH_SIZE)\n#     ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]\n    sample_submission[[f'preds{var}'.format(var)]]=pred\n    print(f'/kaggle/input/g2net-tf-keras-train-test/fold-{var}.h5'.format(var))\n    var=var+1\n    print(f'preds{var}'.format(var))\n    K.clear_session()\n    gc.collect()\n    try:\n        del model\n    except:\n        print('no model here')\n#     try:\n#         del ds_test\n#     except:\n#         print('no ds here')\n#     try:\n#         del ct_test\n#     except:\n#         print('no ct here')\n#     try:\n#         del files_test\n#     except:\n#         print('no files_test')\n    gc.collect","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:25:04.024575Z","iopub.execute_input":"2022-12-19T11:25:04.02532Z","iopub.status.idle":"2022-12-19T11:35:02.3423Z","shell.execute_reply.started":"2022-12-19T11:25:04.025282Z","shell.execute_reply":"2022-12-19T11:35:02.341239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K.clear_session()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:46:12.650883Z","iopub.execute_input":"2022-12-19T12:46:12.651597Z","iopub.status.idle":"2022-12-19T12:46:13.135532Z","shell.execute_reply.started":"2022-12-19T12:46:12.651561Z","shell.execute_reply":"2022-12-19T12:46:13.134619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    del model\nexcept:\n    print('no model here')\ntry:\n    del ds_test\nexcept:\n    print('no ds here')\ntry:\n    del ct_test\nexcept:\n    print('no ct here')\ntry:\n    del files_test\nexcept:\n    print('no files_test')\ngc.collect","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:46:15.33367Z","iopub.execute_input":"2022-12-19T12:46:15.334347Z","iopub.status.idle":"2022-12-19T12:46:15.357153Z","shell.execute_reply.started":"2022-12-19T12:46:15.334309Z","shell.execute_reply":"2022-12-19T12:46:15.356094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"universal_model_targets=pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:36:44.506187Z","iopub.execute_input":"2022-12-19T11:36:44.50655Z","iopub.status.idle":"2022-12-19T11:36:44.512686Z","shell.execute_reply.started":"2022-12-19T11:36:44.506515Z","shell.execute_reply":"2022-12-19T11:36:44.511075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    del sample_submission['\\ttarget']\nexcept:\n    print('not there')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:40:30.114489Z","iopub.execute_input":"2022-12-19T11:40:30.114849Z","iopub.status.idle":"2022-12-19T11:40:30.120309Z","shell.execute_reply.started":"2022-12-19T11:40:30.114818Z","shell.execute_reply":"2022-12-19T11:40:30.119305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:40:33.811368Z","iopub.execute_input":"2022-12-19T11:40:33.811756Z","iopub.status.idle":"2022-12-19T11:40:33.827497Z","shell.execute_reply.started":"2022-12-19T11:40:33.811722Z","shell.execute_reply":"2022-12-19T11:40:33.826633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" sample_submission.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:42:00.597796Z","iopub.execute_input":"2022-12-19T11:42:00.598461Z","iopub.status.idle":"2022-12-19T11:42:00.610981Z","shell.execute_reply.started":"2022-12-19T11:42:00.598423Z","shell.execute_reply":"2022-12-19T11:42:00.610041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets=(sample_submission['preds0']+sample_submission['preds1'])/2#+sample_submission[['preds2']]+sample_submission[['preds3']]+sample_submission[['preds4']])/4","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:42:37.442858Z","iopub.execute_input":"2022-12-19T11:42:37.443435Z","iopub.status.idle":"2022-12-19T11:42:37.453917Z","shell.execute_reply.started":"2022-12-19T11:42:37.44339Z","shell.execute_reply":"2022-12-19T11:42:37.4529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(targets)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:43:59.66732Z","iopub.execute_input":"2022-12-19T11:43:59.667699Z","iopub.status.idle":"2022-12-19T11:43:59.994166Z","shell.execute_reply.started":"2022-12-19T11:43:59.667642Z","shell.execute_reply":"2022-12-19T11:43:59.99314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(targets- sample_submission.target)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:44:46.854842Z","iopub.execute_input":"2022-12-19T11:44:46.855338Z","iopub.status.idle":"2022-12-19T11:44:47.37084Z","shell.execute_reply.started":"2022-12-19T11:44:46.855297Z","shell.execute_reply":"2022-12-19T11:44:47.369842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(style=\"darkgrid\")\n\n\nsns.histplot(data=sample_submission, x=\"preds0\", color=\"skyblue\", label=\"Universal\", kde=True)\nsns.histplot(data=sample_submission, x=\"target\", color=\"red\", label=\"Wave\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds1\", color=\"blue\", label=\"Universal\", kde=True)\nplt.legend() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:50:05.26133Z","iopub.execute_input":"2022-12-19T11:50:05.261721Z","iopub.status.idle":"2022-12-19T11:50:06.967542Z","shell.execute_reply.started":"2022-12-19T11:50:05.261682Z","shell.execute_reply":"2022-12-19T11:50:06.966622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var=5\nTTA=1\nfiles_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\nds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*2)\nct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/2\nfor i in range(5):\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]\n    sample_submission[[f'preds{var}'.format(var)]]=pred\n    print(f'/kaggle/input/g2net-tf-keras-train-test/fold-{var}.h5'.format(var))\n    var=var+1\n    print(f'preds{var}'.format(var))\n    K.clear_session()\n \n    gc.collect()\n    try:\n        del model\n    except:\n        print('no model here')\n#     try:\n#         del ds_test\n#     except:\n#         print('no ds here')\n#     try:\n#         del ct_test\n#     except:\n#         print('no ct here')\n#     try:\n#         del files_test\n#     except:\n#         print('no files_test')\n    gc.collect","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:47:54.658717Z","iopub.execute_input":"2022-12-19T12:47:54.659111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" sample_submission.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:35:30.164979Z","iopub.execute_input":"2022-12-19T12:35:30.165337Z","iopub.status.idle":"2022-12-19T12:35:30.186518Z","shell.execute_reply.started":"2022-12-19T12:35:30.165305Z","shell.execute_reply":"2022-12-19T12:35:30.185639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import rcParams\nsns.set(style=\"darkgrid\")\nrcParams['figure.figsize'] = 15,10\n\nsns.histplot(data=sample_submission, x=\"preds0\", color=\"skyblue\", label=\"Universal\", kde=True)\nsns.histplot(data=sample_submission, x=\"target\", color=\"red\", label=\"Wave\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds1\", color=\"blue\", label=\"Universal\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds5\", color=\"magenta\", label=\"gaussian\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds6\", color=\"cyan\", label=\"gaussian\", kde=True)\n\nplt.legend()\nplt.legend() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:20:32.52409Z","iopub.execute_input":"2022-12-19T12:20:32.524673Z","iopub.status.idle":"2022-12-19T12:20:35.217316Z","shell.execute_reply.started":"2022-12-19T12:20:32.524618Z","shell.execute_reply":"2022-12-19T12:20:35.216239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_submission, x=\"preds0\", color=\"skyblue\", label=\"Universal\", kde=True)\n\nsns.histplot(data=sample_submission, x=\"preds1\", color=\"blue\", label=\"Universal\", kde=True)\n\n\nplt.legend()\nplt.legend() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:24:39.468265Z","iopub.execute_input":"2022-12-19T12:24:39.469071Z","iopub.status.idle":"2022-12-19T12:24:40.714529Z","shell.execute_reply.started":"2022-12-19T12:24:39.469034Z","shell.execute_reply":"2022-12-19T12:24:40.713591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"\nsns.histplot(data=sample_submission, x=\"preds5\", color=\"magenta\", label=\"gaussian\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds6\", color=\"cyan\", label=\"gaussian\", kde=True)\n\nplt.legend()\nplt.legend() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:23:40.703068Z","iopub.execute_input":"2022-12-19T12:23:40.703446Z","iopub.status.idle":"2022-12-19T12:23:42.19152Z","shell.execute_reply.started":"2022-12-19T12:23:40.703414Z","shell.execute_reply":"2022-12-19T12:23:42.190574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsns.histplot(data=sample_submission, x=\"target\", color=\"red\", label=\"Wave\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds5\", color=\"magenta\", label=\"gaussian\", kde=True)\nsns.histplot(data=sample_submission, x=\"preds6\", color=\"cyan\", label=\"gaussian\", kde=True)\n\nplt.legend()\nplt.legend() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:21:31.213486Z","iopub.execute_input":"2022-12-19T12:21:31.213868Z","iopub.status.idle":"2022-12-19T12:21:32.571382Z","shell.execute_reply.started":"2022-12-19T12:21:31.213835Z","shell.execute_reply":"2022-12-19T12:21:32.570044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:29:55.678148Z","iopub.execute_input":"2022-12-19T12:29:55.678584Z","iopub.status.idle":"2022-12-19T12:29:55.689388Z","shell.execute_reply.started":"2022-12-19T12:29:55.67855Z","shell.execute_reply":"2022-12-19T12:29:55.688453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T11:18:28.105751Z","iopub.status.idle":"2022-12-19T11:18:28.106518Z","shell.execute_reply.started":"2022-12-19T11:18:28.106267Z","shell.execute_reply":"2022-12-19T11:18:28.106291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['filter']=sample_submission.probability\nsample_submission['universal_model']=(sample_submission['preds1']+sample_submission['preds0'])/2\nsample_submission['gaussian_model']=(sample_submission['preds6']+sample_submission['preds5'])/2\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:30:42.419768Z","iopub.execute_input":"2022-12-19T12:30:42.420216Z","iopub.status.idle":"2022-12-19T12:30:42.433845Z","shell.execute_reply.started":"2022-12-19T12:30:42.420162Z","shell.execute_reply":"2022-12-19T12:30:42.432719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conditions = [\n    (sample_submission['filter'] < 0.8),\n    (sample_submission['filter'] >= 0.8) \n]\n\nvalues = ['gaussian', 'non_gaussian']\n\nsample_submission['hue'] = np.select(conditions, values)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:32:50.343168Z","iopub.execute_input":"2022-12-19T12:32:50.343524Z","iopub.status.idle":"2022-12-19T12:32:50.355198Z","shell.execute_reply.started":"2022-12-19T12:32:50.343493Z","shell.execute_reply":"2022-12-19T12:32:50.354143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.loc[sample_submission.filter <= 0.8, 'target'] = sample_submission.gaussian_model\n# sample_submission.loc[sample_submission.filter >0.8, 'target'] = sample_submission.universal_model\nsample_submission.loc[sample_submission.hue== 'non_gaussian', 'target'] = sample_submission.universal_model\nsample_submission.loc[sample_submission.hue== 'gaussian', 'target'] = sample_submission.gaussian_model","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:38:14.70945Z","iopub.execute_input":"2022-12-19T12:38:14.709837Z","iopub.status.idle":"2022-12-19T12:38:14.732261Z","shell.execute_reply.started":"2022-12-19T12:38:14.709804Z","shell.execute_reply":"2022-12-19T12:38:14.729784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_submission, x=\"target\", color=\"red\", label=\"Agregated\", kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:39:54.103619Z","iopub.execute_input":"2022-12-19T12:39:54.104677Z","iopub.status.idle":"2022-12-19T12:39:57.278804Z","shell.execute_reply.started":"2022-12-19T12:39:54.10462Z","shell.execute_reply":"2022-12-19T12:39:57.277722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_submission, x=\"gaussian_model\", color=\"red\", label=\"Gaussian\", kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:44:18.617142Z","iopub.execute_input":"2022-12-19T12:44:18.617508Z","iopub.status.idle":"2022-12-19T12:44:19.346017Z","shell.execute_reply.started":"2022-12-19T12:44:18.617477Z","shell.execute_reply":"2022-12-19T12:44:19.345078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=sample_submission, x=\"gaussian_model\", color=\"red\", label=\"Gaussian\", kde=True)\nsns.histplot(data=sample_submission, x=\"universal_model\", color=\"blue\", label=\"Universal\", kde=True)\nsns.histplot(data=sample_submission, x=\"target\", color=\"magenta\", label=\"Agregated\", kde=True)\nplt.legend()\nplt.legend() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:42:59.451199Z","iopub.execute_input":"2022-12-19T12:42:59.451576Z","iopub.status.idle":"2022-12-19T12:43:03.794586Z","shell.execute_reply.started":"2022-12-19T12:42:59.451544Z","shell.execute_reply":"2022-12-19T12:43:03.793577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[['id','target']]=sample_submission[['id','target']]","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:44:46.681254Z","iopub.execute_input":"2022-12-19T12:44:46.681618Z","iopub.status.idle":"2022-12-19T12:44:46.690401Z","shell.execute_reply.started":"2022-12-19T12:44:46.681586Z","shell.execute_reply":"2022-12-19T12:44:46.689365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission1.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:44:51.943088Z","iopub.execute_input":"2022-12-19T12:44:51.943528Z","iopub.status.idle":"2022-12-19T12:44:51.977709Z","shell.execute_reply.started":"2022-12-19T12:44:51.943489Z","shell.execute_reply":"2022-12-19T12:44:51.976825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect","metadata":{"execution":{"iopub.status.busy":"2022-12-19T12:45:26.740561Z","iopub.execute_input":"2022-12-19T12:45:26.741671Z","iopub.status.idle":"2022-12-19T12:45:26.748687Z","shell.execute_reply.started":"2022-12-19T12:45:26.741605Z","shell.execute_reply":"2022-12-19T12:45:26.747711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}