{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\nThe result is obtained in a few steps:\n1. **DatPrep**: The provided data are loaded in and stored locally in pickle format. A time binning is applied (simple averages over time intervals) to reduce storage. This facilitates the analysis of hyperparams etc later, otherwise the data loading largely dominates the training process.\n2. **DatGen**: A Gaussian convolution is applied to preprocessed images to enhance image recognition later (a \"smooth\" time binning let's say). Some random 'white noise' false signals are added to dataset. These are pretty easily recognised but augment dataset\n3. **Model**: The model consists of augmentation layer, torchvision efficientnet CNN and dense relu layers at the end. Data augmentation is simple scale, flip and shift of images.\n4. **Training**: KFold is applied, the training samples are augmented with random white noise false signals","metadata":{}},{"cell_type":"markdown","source":"# Data load and save as .pkl files \nThis step is preparation and takes roughly 20 min with 2 workers or 10 min with 4 workers","metadata":{}},{"cell_type":"code","source":"# packages used\nimport pandas as pd\nimport os\nimport numpy as np\nimport tensorflow as tf\nimport h5py\nimport time\nimport progressbar\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport pickle\nfrom functools import partial\nfrom tqdm import tqdm\nfrom multiprocessing import Pool\n\nimport torch\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:50.918341Z","iopub.execute_input":"2022-12-12T08:18:50.918668Z","iopub.status.idle":"2022-12-12T08:18:57.82984Z","shell.execute_reply.started":"2022-12-12T08:18:50.918635Z","shell.execute_reply":"2022-12-12T08:18:57.828548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# files located:\nPATH = '/kaggle/input/g2net-detecting-continuous-gravitational-waves'\n\n# make storage path\n!mkdir ../temp","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:57.831208Z","iopub.execute_input":"2022-12-12T08:18:57.83431Z","iopub.status.idle":"2022-12-12T08:18:58.780454Z","shell.execute_reply.started":"2022-12-12T08:18:57.834279Z","shell.execute_reply":"2022-12-12T08:18:58.779184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_sample(file_id, timebins=128, t_select=4096, data_type='train', folder='../temp'):\n    '''\n    Load an image, reduce time time dimension and save as .pkl for fast read-in later\n    '''\n    filename = '%s/%s/%s.hdf5' % (PATH, data_type, file_id)\n    sample_data = h5py.File(filename , 'r')\n        \n    frame = np.zeros((2, 360, timebins),dtype=complex)\n    for ind, d in enumerate(['H1','L1']):\n        SFT = np.array(sample_data[file_id][d]['SFTs'])[:,:t_select]*1e22\n        amp_SFT = (np.real(SFT)**2 + np.imag(SFT)**2).astype('float64') # complex amplitude\n        frame[ind] = np.mean(amp_SFT.reshape((360,timebins,-1)), axis=-1)\n    \n    with open(f\"{folder}/{data_type}-{file_id}.pkl\", \"wb\") as f:\n        f.write(pickle.dumps(frame))\n\n# save samples locally by iterating over ids in parallel\ndef parallel_save_samples(file_ids, timebins=128, t_select=4096, data_type='train'):\n    '''\n    Parallel loop over a list of file_ids to save all samples reduced in .pkl format\n    '''\n\n    N = len(file_ids)\n    \n    with Pool(os.cpu_count()) as pool:\n        with tqdm(total=N) as t:\n            for _ in pool.imap_unordered(\n                partial(save_sample, timebins=timebins, t_select=t_select, data_type=data_type), \n                list(file_ids)):\n                t.update(1)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:18:58.783166Z","iopub.execute_input":"2022-12-12T08:18:58.783628Z","iopub.status.idle":"2022-12-12T08:18:58.796639Z","shell.execute_reply.started":"2022-12-12T08:18:58.783583Z","shell.execute_reply":"2022-12-12T08:18:58.795739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read in .csv's\nCSV_PATH = '../input/g2net-detecting-continuous-gravitational-waves/'\ndf_train_raw = pd.read_csv(CSV_PATH + 'train_labels.csv')\ndf_test_raw = pd.read_csv(CSV_PATH + 'sample_submission.csv')\n\n# remove undecided rows from df\ndf_train_raw = df_train_raw[df_train_raw.target>=0]\n\nprint('Preparing train data...')\nparallel_save_samples(df_train_raw.id, data_type='train')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:20:32.098836Z","iopub.execute_input":"2022-12-12T08:20:32.099206Z","iopub.status.idle":"2022-12-12T08:21:58.489357Z","shell.execute_reply.started":"2022-12-12T08:20:32.099175Z","shell.execute_reply":"2022-12-12T08:21:58.48815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Preparing test data...')\nparallel_save_samples(df_test_raw.id, data_type='test')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:21:58.491614Z","iopub.execute_input":"2022-12-12T08:21:58.49196Z","iopub.status.idle":"2022-12-12T08:41:34.931551Z","shell.execute_reply.started":"2022-12-12T08:21:58.491928Z","shell.execute_reply":"2022-12-12T08:41:34.929408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# see if all files have been prepared\n!ls '../temp/' ","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-12T09:29:22.9546Z","iopub.execute_input":"2022-12-12T09:29:22.955533Z","iopub.status.idle":"2022-12-12T09:29:23.941084Z","shell.execute_reply.started":"2022-12-12T09:29:22.955484Z","shell.execute_reply":"2022-12-12T09:29:23.939945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data load function","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils","metadata":{"execution":{"iopub.status.busy":"2022-12-12T09:29:26.343968Z","iopub.execute_input":"2022-12-12T09:29:26.344372Z","iopub.status.idle":"2022-12-12T09:29:26.657952Z","shell.execute_reply.started":"2022-12-12T09:29:26.344335Z","shell.execute_reply":"2022-12-12T09:29:26.656965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.ndimage import gaussian_filter\n\ndef normalize_t(X, norm=None):\n    '''\n    normalize each time slice so that they all have the same mean. \n    If norm=None, np.mean(X) is left invariant after transform and is now the mean of all columns\n    '''\n    \n    N = X.shape[0]\n    \n    if norm is None:\n        norm = np.mean(X)\n    \n    X_out = np.zeros(X.shape)\n    for i in range(N):\n        mt = np.repeat(np.mean(X[i], axis=0, keepdims=True), 360, axis=0)\n        X_out[i] = (norm/mt) * X[i]\n    \n    return X_out\n    \ndef apply_gaussian_kernel(X, sigma):\n    \n    '''\n    Apply convolution with a gaussian kernel with variance sigma. \n    '''\n    \n    N = X.shape[0]\n    \n    X_out = np.zeros(X.shape)\n    for i in range(N):\n        dat = gaussian_filter(X[i], sigma=sigma)\n        X_out[i] = dat\n    \n    return X_out\n\nclass Dataset(Dataset):\n    \"\"\"\n    dataset = Dataset(data_type, df)\n\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 128\n      y (np.float32): label 0 or 1\n      Entries in df with id='random_false' will be random white noise images\n    \"\"\"\n    def __init__(self, data_type, df, normalize_time_slices=True, sigma=None):\n        self.data_type = data_type\n        self.df = df\n        self.sigma = sigma\n        self.normalize_time_slices = normalize_time_slices\n        \n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, i):\n        \"\"\"\n        i (int): get ith data\n        \"\"\"\n        \n        r = self.df.iloc[i]\n        y = np.float32(r.target)\n        file_id = r.id\n        \n        if file_id == \"random_false\":\n            # generate random false sample with sampling some gaussian noise\n            x = np.random.normal(0., 1., size=(2,360,128))\n        \n        else:\n            with open(f\"../temp/{self.data_type}-{file_id}.pkl\", \"rb\") as rf:\n                x = np.real(pickle.loads(rf.read()))\n            \n        if self.normalize_time_slices:\n            x = normalize_t(x)\n        \n        if self.sigma:\n            x = apply_gaussian_kernel(x, self.sigma)\n        \n        # normalize\n        x = (x-np.min(x)) / (np.max(x)- np.min(x))\n        return x.astype(np.double), y","metadata":{"execution":{"iopub.status.busy":"2022-12-12T09:29:27.012411Z","iopub.execute_input":"2022-12-12T09:29:27.012871Z","iopub.status.idle":"2022-12-12T09:29:27.235885Z","shell.execute_reply.started":"2022-12-12T09:29:27.01283Z","shell.execute_reply":"2022-12-12T09:29:27.234854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# quick test what a frame looks like\ni = 10 # entry\npanel = 0 # H1 or F1 panel\nsigma = (1., 3.) # Gaussian convolution (sigma_freq, sigma_t)\n\nds = Dataset('train', df_train_raw, sigma=sigma)\nx = ds[i][0][panel]\ny = int(ds[i][1])\n\n\nplt.pcolormesh(x, cmap=cm.gray)\nplt.title(f'target: {y}')\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:11:48.163756Z","iopub.execute_input":"2022-12-12T10:11:48.164433Z","iopub.status.idle":"2022-12-12T10:11:48.453599Z","shell.execute_reply.started":"2022-12-12T10:11:48.164399Z","shell.execute_reply":"2022-12-12T10:11:48.45261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"def Augmentation(max_shift=0.05, max_scaling=0.05, flip=True):\n    '''\n    Apply random operation\n    max_shift: shift image in freq dim\n    max_scaling: scale image in freq dim\n    flip: randomly flip image\n    \n    returns pipeline\n    '''\n    t = []\n    t.append(transforms.RandomAffine(degrees=0.,translate=(0., max_shift), scale=(1.-max_scaling, 1.+max_scaling)))\n   \n    if flip:\n        t.append(transforms.RandomVerticalFlip(p=0.5))\n                 \n    return transforms.Compose(t)\n        ","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:11:51.014354Z","iopub.execute_input":"2022-12-12T10:11:51.0154Z","iopub.status.idle":"2022-12-12T10:11:51.021732Z","shell.execute_reply.started":"2022-12-12T10:11:51.015352Z","shell.execute_reply":"2022-12-12T10:11:51.020689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# quick test to see how it produces samples for input\ni = 10\npanel = 1\nsigma = (3., 10.)\nds = Dataset('train', df_train_raw, sigma=sigma)\n\nt = torch.from_numpy(ds[i][0])\naug = Augmentation()\n\nplt.pcolormesh(aug(t).numpy()[0], cmap=cm.gray)\nplt.title(f'target: {ds[i][1]}')\nplt.colorbar()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:11:52.976972Z","iopub.execute_input":"2022-12-12T10:11:52.977336Z","iopub.status.idle":"2022-12-12T10:11:53.433523Z","shell.execute_reply.started":"2022-12-12T10:11:52.977306Z","shell.execute_reply":"2022-12-12T10:11:53.432594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EfficientNet","metadata":{"execution":{"iopub.status.busy":"2022-11-30T15:09:10.575679Z","iopub.execute_input":"2022-11-30T15:09:10.57617Z","iopub.status.idle":"2022-11-30T15:09:10.584461Z","shell.execute_reply.started":"2022-11-30T15:09:10.57613Z","shell.execute_reply":"2022-11-30T15:09:10.581995Z"}}},{"cell_type":"code","source":"# for image classification\n!pip install timm\n!pip install torchsummary","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-12T10:12:01.840155Z","iopub.execute_input":"2022-12-12T10:12:01.840507Z","iopub.status.idle":"2022-12-12T10:12:24.130906Z","shell.execute_reply.started":"2022-12-12T10:12:01.840477Z","shell.execute_reply":"2022-12-12T10:12:24.129705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nfrom torchsummary import summary\nimport timm\n\n# on gpu (otherwise change to 'cpu')\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") \n\n# the model: consists of data augmentation layer, efficientnet CNN, and final linear layer with dropout\nclass Model(nn.Module):\n    def __init__(self, name, *, dropout=0.5, pretrained=False, transform_da=None, mean_detectors=False, num_dense=[1024]):\n        \"\"\"\n        name (str): timm model name, e.g. tf_efficientnet_b2_ns\n        \"\"\"\n        super().__init__()\n\n        # Use timm\n        in_chans = 1 if mean_detectors else 2\n        model = timm.create_model(name, pretrained=pretrained, in_chans=in_chans)\n\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity()\n    \n        self.fc = nn.Linear(n_features, 1)\n        self.model = model\n        \n        self.transform_da = transform_da\n        self.mean_detectors = mean_detectors\n        \n        # final dense layers after CNN\n        modules = []\n        features = [n_features] + num_dense\n        \n        if len(features) > 1:\n            for il in range(len(features)-1):\n                modules.append(nn.Linear(features[il],features[il+1]))\n                modules.append(nn.ReLU())\n                modules.append(nn.Dropout(p=dropout))\n        \n        # last layer is single output for prediction\n        modules.append(nn.Linear(features[-1], 1))\n        \n        self.dense = nn.Sequential(*modules)\n\n    def forward(self, x):\n        \n        x = x.float()\n        \n        if self.mean_detectors:\n            x = torch.mean(x, dim=1, keepdim=True) \n        \n        # data augmentation transforms, e.g. random rescale or shift\n        if self.transform_da:\n            x = self.transform_da(x)\n\n        # Efficientnet layer\n        x = self.model(x)\n\n        # final dense network\n        x = self.dense(x)\n        return x\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:12:24.133919Z","iopub.execute_input":"2022-12-12T10:12:24.134339Z","iopub.status.idle":"2022-12-12T10:12:24.975927Z","shell.execute_reply.started":"2022-12-12T10:12:24.134299Z","shell.execute_reply":"2022-12-12T10:12:24.974729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and evaluate","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef evaluate(model, loader_val, *, compute_score=True, pbar=None, criterion=nn.BCEWithLogitsLoss()):\n    \"\"\"\n    Predict and compute loss and score\n    \"\"\"\n    tb = time.time()\n    was_training = model.training\n    model.eval()\n\n    loss_sum = 0.0\n    n_sum = 0\n    y_all = []\n    y_pred_all = []\n\n    if pbar is not None:\n        pbar = tqdm(desc='Predict', nrows=78, total=pbar)\n\n    for img, y in loader_val:\n        n = y.size(0)\n        img = img.to(device)\n        y = y.to(device)\n\n        with torch.no_grad():\n            y_pred = model(img.to(device))\n        loss = criterion(y_pred.view(-1), y)\n\n        n_sum += n\n        loss_sum += n * loss.item()\n\n        y_all.append(y.cpu().detach().numpy())\n        y_pred_all.append(y_pred.sigmoid().squeeze().cpu().detach().numpy())\n\n        if pbar is not None:\n            pbar.update(len(img))\n        \n        del loss, y_pred, img, y\n\n    loss_val = loss_sum / n_sum\n\n    y = np.concatenate(y_all)\n    y_pred = np.concatenate(y_pred_all)\n\n    score = roc_auc_score(y, y_pred) if compute_score else None\n\n    ret = {'loss': loss_val,\n           'score': score,\n           'y': y,\n           'y_pred': y_pred,\n           'time': time.time() - tb}\n    \n    model.train(was_training)  # back to train from eval if necessary\n\n    return ret","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:12:24.978156Z","iopub.execute_input":"2022-12-12T10:12:24.978771Z","iopub.status.idle":"2022-12-12T10:12:25.763498Z","shell.execute_reply.started":"2022-12-12T10:12:24.978729Z","shell.execute_reply":"2022-12-12T10:12:25.762552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\n\n\n\nmodel_name = 'tf_efficientnet_b1_ns'\nnfold = 5\nkfold = StratifiedKFold(n_splits=nfold, random_state=42, shuffle=True)\n\nrandom_falses = 200 # extra random false samples added to training set\nsigma_GaussConv = (1.,3.) # sigma for preparatory Gauss conv (blurring)\nrandom_translation = 0.05 # random freq shift\nrandom_scaling = 0.05 # random scaling\nmean_detectors = False # take mean of both detectors as input image\nnum_dense = [1024] # Dense relu layers at the end with dropout and L2 regularization\nweight_decay = 5e-2 # L2 regularize\ndropout = 0.5 # dropout\nmax_grad_norm = 500 \n\nepochs = 20\nbatch_size = 16\nnum_workers = 2\n\nlr_max = 4e-4\nepochs_warmup = 4.0\n\ndataset = Dataset('train', df_train_raw, sigma=sigma_GaussConv)\n\n# data augmentation\naug = Augmentation(max_shift=random_translation, max_scaling=random_scaling, flip=True)\n\nfor ifold, (idx_train, idx_test) in enumerate(kfold.split(dataset, df_train_raw['target'])):\n    print('Fold %d/%d' % (ifold+1, nfold))\n    torch.manual_seed(42 + ifold + 1)\n    \n    # concat fake falses (random white noise)\n    df_train_local = pd.concat([df_train_raw.iloc[idx_train], \n                      pd.DataFrame([['random_false', 0.]], index=range(random_falses), columns=list(['id', 'target']))],\n                     ignore_index=True)\n    \n    # shuffle again\n    df_train_local = df_train_local.sample(frac=1).reset_index(drop=True)\n    \n    # get balances pos over neg samples and get loss function with correct weights\n    num_positives = torch.sum(torch.Tensor(df_train_local.target), dim=0)\n    num_negatives = len(df_train_local.target) - num_positives\n    pos_weight  = num_negatives / num_positives\n    criterion = nn.BCEWithLogitsLoss(pos_weight = pos_weight )\n\n    # Train - val split\n    dataset_train = Dataset('train', df_train_local, sigma=sigma_GaussConv)\n    dataset_val = Dataset('train', df_train_raw.iloc[idx_test], sigma=sigma_GaussConv)\n\n    loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size,\n                     num_workers=num_workers, pin_memory=True, shuffle=True, drop_last=True)\n    loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=batch_size,\n                     num_workers=num_workers, pin_memory=True)\n\n    # Model and optimizer\n    model = Model(model_name, \n                  pretrained=True, \n                  transform_da=aug,\n                  num_dense=num_dense,\n                  dropout=dropout, \n                  mean_detectors=mean_detectors)\n    model.to(device)\n    model.train()\n    \n    # print summary\n    # summary(model, (2, 360, 128))\n\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr_max, weight_decay=weight_decay)\n\n    # Learning-rate schedule\n    nbatch = len(loader_train)\n    warmup = epochs_warmup * nbatch  # number of warmup steps\n    nsteps = epochs * nbatch        # number of total steps\n\n    scheduler = CosineLRScheduler(optimizer,\n                  warmup_t=warmup, warmup_lr_init=0.0, warmup_prefix=True, # 1 epoch of warmup\n                  t_initial=(nsteps - warmup), lr_min=1e-7)                # 3 epochs of cosine\n    \n    time_val = 0.0\n    lrs = []\n\n    tb = time.time()\n    print('Epoch   loss          score          lr')\n    for iepoch in range(epochs):\n        loss_sum = 0.0\n        n_sum = 0\n\n        # Train\n        for ibatch, (img, y) in enumerate(loader_train):\n            n = y.size(0)\n            img = img.to(device)\n            y = y.to(device)\n\n            optimizer.zero_grad()\n\n            y_pred = model(img)\n            loss = criterion(y_pred.view(-1), y)\n\n            loss_train = loss.item()\n            loss_sum += n * loss_train\n            n_sum += n\n\n            loss.backward()\n\n            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),\n                                                       max_grad_norm)\n            optimizer.step()\n            \n            scheduler.step(iepoch * nbatch + ibatch + 1)\n            lrs.append(optimizer.param_groups[0]['lr'])            \n\n        # Evaluate\n        res_train = evaluate(model, loader_train)\n        res_val = evaluate(model, loader_val)\n        time_val += res_val['time']\n        loss_train = loss_sum / n_sum\n        lr_now = optimizer.param_groups[0]['lr']\n        dt = (time.time() - tb) / 60\n        print('Epoch %d %.4f %.4f %.4f %.4f  %.2e  %.2f min' %\n              (iepoch + 1, res_train['loss'], res_val['loss'], res_train['score'], res_val['score'], lr_now, dt))\n\n    dt = time.time() - tb\n    print('Training done %.2f min total, %.2f min val' % (dt / 60, time_val / 60))\n\n    # Save model\n    ofilename = 'model%d.pytorch' % ifold\n    torch.save(model.state_dict(), ofilename)\n    print(ofilename, 'written')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-12T10:19:40.229741Z","iopub.execute_input":"2022-12-12T10:19:40.230109Z","iopub.status.idle":"2022-12-12T10:39:06.419321Z","shell.execute_reply.started":"2022-12-12T10:19:40.230078Z","shell.execute_reply":"2022-12-12T10:39:06.417264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict and submit","metadata":{}},{"cell_type":"code","source":"submit = pd.read_csv(CSV_PATH + '/sample_submission.csv')\nsubmit['target'] = 0.\n\nfor ifold in range(nfold):\n    \n    print('Fold %d/%d' % (ifold+1, nfold))\n    \n    model = Model(model_name, pretrained=False, mean_detectors=mean_detectors)\n    model.transform_da = None\n    \n    filename = f'model{ifold}.pytorch'\n    model.to(device)\n    model.load_state_dict(torch.load(filename, map_location=device))\n    model.eval()\n\n    dataset_test = Dataset('test', submit, sigma=sigma_GaussConv)\n    loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=64,\n                                            num_workers=num_workers, pin_memory=True)\n\n    test = evaluate(model, loader_test, compute_score=False, pbar=len(submit))\n\n    submit['target'] += test['y_pred'] / nfold\nsubmit.to_csv('submission.csv', index=False)\nprint('target range [%.2f, %.2f]' % (submit['target'].min(), submit['target'].max()))","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:39:06.423238Z","iopub.execute_input":"2022-12-12T10:39:06.423922Z","iopub.status.idle":"2022-12-12T10:45:35.331427Z","shell.execute_reply.started":"2022-12-12T10:39:06.423886Z","shell.execute_reply":"2022-12-12T10:45:35.328298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}