{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade torchmetrics efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:01:08.827425Z","iopub.execute_input":"2022-10-24T17:01:08.827898Z","iopub.status.idle":"2022-10-24T17:01:20.044658Z","shell.execute_reply.started":"2022-10-24T17:01:08.827865Z","shell.execute_reply":"2022-10-24T17:01:20.043586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nfrom pathlib import Path\nfrom dataclasses import dataclass\nfrom typing import Optional, Any, Union, Dict, NamedTuple\n\nimport h5py\nimport wandb\nimport torch\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pytorch_lightning as pl\nfrom torch import nn, Tensor\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\nfrom torchmetrics.classification import BinaryAUROC\nfrom efficientnet_pytorch import EfficientNet\nimport torchvision.transforms as T\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames[:5]:\n        print(os.path.join(dirname, filename))\n        \nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\n\n# I have saved my API token with \"wandb_api\" as Label. \n# If you use some other Label make sure to change the same below. \nwandb_api = user_secrets.get_secret(\"wandb_api\") \n\nwandb.login(key=wandb_api)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\npl.seed_everything(1234)\ntorch.backends.cudnn.benchmark = True","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-24T17:01:20.047167Z","iopub.execute_input":"2022-10-24T17:01:20.049394Z","iopub.status.idle":"2022-10-24T17:01:23.431657Z","shell.execute_reply.started":"2022-10-24T17:01:20.049349Z","shell.execute_reply":"2022-10-24T17:01:23.430684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Batch(NamedTuple):\n    file_id: str\n    l1_amps: Tensor\n    h1_amps: Tensor\n    targets: int\n\n\nclass HDF5Dataset(torch.utils.data.Dataset):\n    \"\"\"Dataset for HDF5 files.\"\"\"\n\n    def __init__(self, root: Path, stage: str = \"train\", max_length: int = 8000):\n        super().__init__()\n        self._root = root\n        self._stage = stage\n        self._max_length = max_length\n\n        if stage == \"test\":\n            self.transforms = nn.Sequential(\n                T.Resize((512, 512)),\n            )\n            self._samples = pd.read_csv(self._root / \"sample_submission.csv\")\n\n        else:\n            self.transforms = nn.Sequential(\n                T.Resize((512, 512)),\n                T.RandomAutocontrast(),\n                T.GaussianBlur(kernel_size=(5, 9), sigma=(0.1, 5)),\n                T.RandomRotation(degrees=(0, 180)),\n                T.RandomAffine(\n                    degrees=(30, 70), translate=(0.1, 0.3), scale=(0.5, 0.75)\n                ),\n            )\n            df = pd.read_csv(self._root / \"train_labels.csv\")\n            self._samples = df.loc[df[\"target\"] != -1]\n\n    def __getitem__(self, idx):\n        file_id, target = self._samples.iloc[idx].values\n        \n        with h5py.File(self._root / self._stage / f\"{file_id}.hdf5\") as file:\n\n            \n            \n            # taken directly from https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification\n            # this is a much better preprocessing workflow than what I was doing.\n            l1_amplitude = np.array(file[file_id][\"L1\"][\"SFTs\"])[:, :4096] * 1e22\n            l1_amplitude = l1_amplitude.real**2 + l1_amplitude.imag**2  # power\n            l1_amplitude /= np.mean(l1_amplitude)  # normalize\n            l1_amplitude = np.mean(\n                l1_amplitude.reshape(360, 128, 32), axis=2\n            )  # compress 4096 -> 128\n\n            h1_amplitude = np.array(file[file_id][\"H1\"][\"SFTs\"])[:, :4096] * 1e22\n            h1_amplitude = h1_amplitude.real**2 + h1_amplitude.imag**2  # power\n            h1_amplitude /= np.mean(h1_amplitude)  # normalize\n            h1_amplitude = np.mean(\n                h1_amplitude.reshape(360, 128, 32), axis=2\n            )  # compress 4096 -> 128\n\n            target = torch.tensor([target]).float()\n\n        return Batch(\n            file_id=file_id,\n            h1_amps=torch.from_numpy(h1_amplitude),\n            l1_amps=torch.from_numpy(l1_amplitude),\n            targets=target,\n        )\n\n    def __len__(self):\n        return len(self._samples)\n\n\nclass G2NetDataModule(pl.LightningDataModule):\n    \"\"\"DataModule for managing datasets.\"\"\"\n\n    def __init__(\n        self,\n        path: Path,\n        num_workers: int = 8,\n        batch_size: int = 32,\n        max_length: int = 8000,\n    ):\n        super().__init__()\n        self._path = path\n        self._num_workers = num_workers\n        self._batch_size = batch_size\n        self._max_length = max_length\n\n    def setup(self, stage: Optional[str] = \"fit\") -> None:\n        \"\"\"Setup datasets.\"\"\"\n\n        dataset = HDF5Dataset(self._path, max_length=self._max_length)\n        train_split = int(np.floor(len(dataset) * 0.8))\n        val_split = int(np.ceil(len(dataset) * 0.2))\n\n        assert (train_split + val_split) == len(dataset)\n\n        train, val = torch.utils.data.random_split(\n            dataset, lengths=(train_split, val_split)\n        )\n        self._train_dataset = train\n        self._val_dataset = val\n        self._test_dataset = HDF5Dataset(\n            self._path, \"test\", max_length=self._max_length\n        )\n\n    def train_dataloader(self) -> DataLoader:\n        \"\"\"Train dataloader.\"\"\"\n        return DataLoader(\n            self._train_dataset,\n            batch_size=self._batch_size,\n            num_workers=self._num_workers,\n            shuffle=True,\n        )\n\n    def val_dataloader(self) -> DataLoader:\n        \"\"\"Validation dataloader.\"\"\"\n        return DataLoader(\n            self._val_dataset,\n            batch_size=self._batch_size,\n            num_workers=self._num_workers,\n            shuffle=False,\n        )\n\n    def predict_dataloader(self) -> DataLoader:\n        \"\"\"Predict dataloader.\"\"\"\n        return DataLoader(\n            self._test_dataset,\n            batch_size=1,\n            num_workers=self._num_workers,\n            shuffle=False,\n        )\n","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:01:23.433084Z","iopub.execute_input":"2022-10-24T17:01:23.433448Z","iopub.status.idle":"2022-10-24T17:01:24.89174Z","shell.execute_reply.started":"2022-10-24T17:01:23.43341Z","shell.execute_reply":"2022-10-24T17:01:24.890833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(pl.LightningModule):\n    \"\"\"Simple Model.\"\"\"\n\n    def __init__(self, in_channels=1, out_channels=1):\n        super().__init__()\n        self.save_hyperparameters()\n\n        self.auc = BinaryAUROC()\n        self.encoder = EfficientNet.from_pretrained(\"efficientnet-b0\")\n        self.flatten = nn.Flatten()\n\n        self.out_proj = nn.Linear(1000, out_channels)\n\n    def forward(self, batch):\n        \"\"\"Forward Pass.\"\"\"\n        inputs = torch.concat(\n            [batch.h1_amps, batch.l1_amps],\n            dim=1,\n        ).unsqueeze(1)\n        inputs = torch.tile(inputs, (1, 3, 1, 1))\n        x = self.encoder(inputs)\n        x = self.flatten(x)\n        x = self.out_proj(x)\n        return x\n\n    def training_step(\n        self, batch: Batch, batch_idx: Any\n    ) -> Union[Tensor, Dict[str, Any]]:\n        \"\"\"Train step.\"\"\"\n        logits = self(batch)\n        loss = F.binary_cross_entropy_with_logits(logits, batch.targets)\n        auc = self.auc(logits, batch.targets)\n\n        self.log(\"train_loss\", loss)\n        self.log(\"train_auc\", auc)\n\n        return loss\n\n    def validation_step(\n        self, batch: Batch, batch_idx: Any\n    ) -> Union[Tensor, Dict[str, Any]]:\n        \"\"\"Val step.\"\"\"\n        logits = self(batch)\n        loss = F.binary_cross_entropy_with_logits(logits, batch.targets)\n        auc = self.auc(logits, batch.targets)\n\n        self.log(\"val_loss\", loss)\n        self.log(\"val_auc\", auc)\n\n        return loss\n\n    def predict_step(self, batch, batch_idx):\n        \"\"\"Predict step\"\"\"\n        logits = self(batch)\n        probs = torch.sigmoid(logits)\n\n        return {\"id\": batch.file_id[0], \"target\": probs.item()}\n\n    def configure_optimizers(self) -> torch.optim.Optimizer:\n        \"\"\"Set optimizer.\"\"\"\n        return torch.optim.AdamW(self.parameters(), lr=5e-5)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:01:24.892999Z","iopub.execute_input":"2022-10-24T17:01:24.893348Z","iopub.status.idle":"2022-10-24T17:01:26.343381Z","shell.execute_reply.started":"2022-10-24T17:01:24.893313Z","shell.execute_reply":"2022-10-24T17:01:26.342445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = Path(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\")\n\nmodel = Model()\ndatamodule = G2NetDataModule(root, num_workers=2, batch_size=8)\ncheckpoint_callback = pl.callbacks.ModelCheckpoint(\n    dirpath=\"./\", filename=\"{epoch}-{val_loss:.2f}\"\n)\nlogger = pl.loggers.WandbLogger(\n    project=\"g2net-2022\",\n    log_model=\"all\",\n    save_dir=\"./\",\n)\n\ntrainer = pl.Trainer(\n    default_root_dir=\"./\",\n    max_epochs=5,\n    accelerator=\"auto\",\n    devices=\"1\",\n    precision=16 if torch.cuda.is_available() else 32,\n    callbacks=checkpoint_callback,\n    logger=logger,\n)\ntrainer.fit(model, datamodule=datamodule)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:01:26.345872Z","iopub.execute_input":"2022-10-24T17:01:26.346527Z","iopub.status.idle":"2022-10-24T17:02:47.611192Z","shell.execute_reply.started":"2022-10-24T17:01:26.346485Z","shell.execute_reply":"2022-10-24T17:02:47.6102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = trainer.predict(model, datamodule=datamodule)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:02:47.612743Z","iopub.execute_input":"2022-10-24T17:02:47.613103Z","iopub.status.idle":"2022-10-24T17:03:29.481235Z","shell.execute_reply.started":"2022-10-24T17:02:47.613059Z","shell.execute_reply":"2022-10-24T17:03:29.480166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame.from_records(predictions).fillna(0.0)\ndf.to_csv(\"submission.csv\", index=0)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-24T17:03:29.482969Z","iopub.execute_input":"2022-10-24T17:03:29.483357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"submission.csv\")\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}