{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":9175762,"sourceType":"datasetVersion","datasetId":5545375}],"dockerImageVersionId":30747,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport scipy.stats\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import r2_score, mean_squared_error\nimport itertools","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-10T11:45:13.926289Z","iopub.execute_input":"2024-08-10T11:45:13.926819Z","iopub.status.idle":"2024-08-10T11:45:16.517663Z","shell.execute_reply.started":"2024-08-10T11:45:13.926771Z","shell.execute_reply":"2024-08-10T11:45:16.516376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/test_adc_info.csv',\n                           index_col='planet_id')\naxis_info = pd.read_parquet('/kaggle/input/ariel-data-challenge-2024/axis_info.parquet')","metadata":{"execution":{"iopub.status.busy":"2024-08-10T11:45:16.520025Z","iopub.execute_input":"2024-08-10T11:45:16.52066Z","iopub.status.idle":"2024-08-10T11:45:16.55355Z","shell.execute_reply.started":"2024-08-10T11:45:16.520618Z","shell.execute_reply":"2024-08-10T11:45:16.552318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_linear_corr(linear_corr,clean_signal):\n    linear_corr = np.flip(linear_corr, axis=0)\n    for x, y in itertools.product(\n                range(clean_signal.shape[1]), range(clean_signal.shape[2])\n            ):\n        poli = np.poly1d(linear_corr[:, x, y])\n        clean_signal[:, x, y] = poli(clean_signal[:, x, y])\n    return clean_signal\n\ndef clean_dark(signal, dead, dark, dt):\n    dark = np.ma.masked_where(dead, dark)\n    dark = np.tile(dark, (signal.shape[0], 1, 1))\n\n    signal -= dark* dt[:, np.newaxis, np.newaxis]\n    return signal\n\ndef preproc(dataset, adc_info, sensor, binning = 15):\n    cut_inf, cut_sup = 39, 321\n    sensor_sizes_dict = {\"AIRS-CH0\":[[11250, 32, 356], [1, 32, cut_sup-cut_inf]], \"FGS1\":[[135000, 32, 32], [1, 32, 32]]}\n    binned_dict = {\"AIRS-CH0\":[11250 // binning // 2, 282], \"FGS1\":[135000 // binning // 2]}\n    linear_corr_dict = {\"AIRS-CH0\":(6, 32, 356), \"FGS1\":(6, 32, 32)}\n    planet_ids = adc_info.index\n    \n    feats = []\n    for i, planet_id in tqdm(list(enumerate(planet_ids))):\n        signal = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/{planet_id}/{sensor}_signal.parquet').to_numpy()\n        dark_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dark.parquet', engine='pyarrow').to_numpy()\n        dead_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/dead.parquet', engine='pyarrow').to_numpy()\n        flat_frame = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/flat.parquet', engine='pyarrow').to_numpy()\n        linear_corr = pd.read_parquet(f'/kaggle/input/ariel-data-challenge-2024/{dataset}/' + str(planet_id) + '/' + sensor + '_calibration/linear_corr.parquet').values.astype(np.float64).reshape(linear_corr_dict[sensor])\n\n        signal = signal.reshape(sensor_sizes_dict[sensor][0]) \n        gain = adc_info[f'{sensor}_adc_gain'].values[i]\n        offset = adc_info[f'{sensor}_adc_offset'].values[i]\n        signal = signal / gain + offset\n        \n        if sensor != \"FGS1\":\n            signal = signal[:, :, cut_inf:cut_sup] #11250 * 32 * 282\n            dt = axis_info['AIRS-CH0-integration_time'].dropna().values\n            linear_corr = linear_corr[:, :, cut_inf:cut_sup]\n            dark_frame = dark_frame[:, cut_inf:cut_sup]\n            dead_frame = dead_frame[:, cut_inf:cut_sup]\n            flat_frame = flat_frame[:, cut_inf:cut_sup]\n        else:\n            dt = np.ones(len(signal))*0.1 \n            \n        linear_corr_signal = apply_linear_corr(linear_corr, signal)\n        signal = clean_dark(signal, dark_frame, dark_frame, dt)\n        \n        flat = flat_frame.reshape(sensor_sizes_dict[sensor][1])\n        flat[dead_frame.reshape(sensor_sizes_dict[sensor][1])] = np.nan\n        signal = signal / flat\n        \n        if sensor == \"FGS1\":\n            signal = signal.reshape((sensor_sizes_dict[sensor][0][0], sensor_sizes_dict[sensor][0][1]*sensor_sizes_dict[sensor][0][2]))\n        \n        mean_signal = np.nanmean(signal, axis=1) # mean over the 32*32(FGS1) or 32(CH0) pixels\n        cds_signal = (mean_signal[1::2] - mean_signal[0::2])\n        binned = np.zeros((binned_dict[sensor]))\n        for j in range(cds_signal.shape[0] // binning):\n            binned[j] = cds_signal[j*binning:j*binning+binning].mean(axis=0)\n                   \n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0],1))\n            \n        feats.append(binned)\n        \n    return np.stack(feats)\n    \npre_test = np.concatenate([preproc('test', test_adc_info, \"FGS1\", 15*12), preproc('test', test_adc_info, \"AIRS-CH0\", 15)], axis=2)","metadata":{"execution":{"iopub.status.busy":"2024-08-10T11:45:16.555045Z","iopub.execute_input":"2024-08-10T11:45:16.555532Z","iopub.status.idle":"2024-08-10T11:45:24.122646Z","shell.execute_reply.started":"2024-08-10T11:45:16.555474Z","shell.execute_reply":"2024-08-10T11:45:24.121362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def phase_detector(signal):\n    phase1, phase2 = None, None\n    best_drop = 0\n    for i in range(50,150):        \n        t1 = signal[i:i+20].max() - signal[i:i+20].min()\n        if t1 > best_drop:\n            phase1 = i+20+5\n            best_drop = t1\n    \n    best_drop = 0\n    for i in range(200,300):\n        t1 = signal[i:i+20].max() - signal[i:i+20].min()\n        if t1 > best_drop:\n            phase2 = i-5\n            best_drop = t1\n    \n    return phase1, phase2\n\ntest = pre_test.copy()\nfor i in range(len(test_adc_info)):\n    p1,p2 = phase_detector(pre_test[i,:,1:].mean(axis=1))\n    test[i] = (test[i] - pre_test[i,p1:p2].mean(axis=0)) / pre_test[i,list(range(p1-40)) + list(range(p2+40,375))].mean(axis=0) * 1000.0","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom fastprogress import master_bar, progress_bar\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\nimport torchvision.models as models\nfrom functools import partial\n\ndef create_model_mnet2():\n    model = models.mobilenet_v3_small(dropout=0.0, norm_layer = nn.Identity)\n    model.features[0][0] = nn.Conv2d(3, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n    model.classifier[3] = nn.Linear(in_features=1024, out_features=283*3, bias=True)\n    return model\n\nclass ImpModel(torch.nn.Module):\n    def __init__(self):\n        super(ImpModel, self).__init__()\n\n        self.filter = nn.Sequential(\n            nn.Conv2d(1, 3, kernel_size=(3,1), stride = (2,1), bias=False),\n            nn.LeakyReLU()\n        )\n        self.model_1d = create_model_mnet2()\n        \n    def forward(self, x):\n        x = self.filter(x)\n        x = self.model_1d(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-08-10T11:45:24.125156Z","iopub.execute_input":"2024-08-10T11:45:24.125566Z","iopub.status.idle":"2024-08-10T11:45:29.344008Z","shell.execute_reply.started":"2024-08-10T11:45:24.125525Z","shell.execute_reply":"2024-08-10T11:45:29.342911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.zeros((len(test), 283))\nsigmas = np.zeros((len(test), 283))\n\nval_dataset = torch.utils.data.TensorDataset(torch.from_numpy(test).unsqueeze(1).float(), torch.zeros(test.shape).float())\nvalidation_loader = torch.utils.data.DataLoader(val_dataset, batch_size=32, shuffle=False)\n\npaths = [\n    '/kaggle/input/ariel-purple-hat/purple_hat_0',\n    '/kaggle/input/ariel-purple-hat/purple_hat_1',\n    '/kaggle/input/ariel-purple-hat/purple_hat_2',\n    '/kaggle/input/ariel-purple-hat/purple_hat_3',\n    '/kaggle/input/ariel-purple-hat/purple_hat_4',\n]\nfor i in range(5):\n    model = ImpModel().cuda()\n    model.load_state_dict(torch.load(paths[i]))\n    model.eval()\n        \n    v_offset = 0\n    with torch.no_grad():        \n        for j, vdata in enumerate(validation_loader):\n            vinputs, vlabels = vdata\n            voutputs = model(vinputs.cuda()).reshape((vinputs.shape[0], 283, 3))\n            preds[v_offset:v_offset+len(vinputs)] += voutputs[:,:,1].detach().cpu().numpy() * 0.2\n            sigmas[v_offset:v_offset+len(vinputs)] += (voutputs[:,:,2].detach().cpu().numpy() - voutputs[:,:,0].detach().cpu().numpy()) * 0.2\n            v_offset += len(vinputs)\n            \n    del model","metadata":{"execution":{"iopub.status.busy":"2024-08-10T11:45:29.356085Z","iopub.execute_input":"2024-08-10T11:45:29.356494Z","iopub.status.idle":"2024-08-10T11:45:30.661709Z","shell.execute_reply.started":"2024-08-10T11:45:29.356461Z","shell.execute_reply":"2024-08-10T11:45:30.660627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/sample_submission.csv')\nsubmission = pd.DataFrame(np.concatenate([preds.clip(0),(sigmas.clip(0)*2.0).clip(1e-5)], axis=1), columns=ss.columns[1:])\nsubmission.index = test_adc_info.index\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-08-10T11:45:30.663099Z","iopub.execute_input":"2024-08-10T11:45:30.663557Z","iopub.status.idle":"2024-08-10T11:45:30.70501Z","shell.execute_reply.started":"2024-08-10T11:45:30.663515Z","shell.execute_reply":"2024-08-10T11:45:30.703852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-08-10T11:45:30.706764Z","iopub.execute_input":"2024-08-10T11:45:30.707574Z","iopub.status.idle":"2024-08-10T11:45:30.738916Z","shell.execute_reply.started":"2024-08-10T11:45:30.70753Z","shell.execute_reply":"2024-08-10T11:45:30.737637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}