{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70367,"databundleVersionId":9188054,"sourceType":"competition"},{"sourceId":205066650,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport pickle\nimport multiprocessing\nimport math\nfrom scipy.optimize import curve_fit\nimport scipy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-04T15:57:01.682293Z","iopub.execute_input":"2024-11-04T15:57:01.682686Z","iopub.status.idle":"2024-11-04T15:57:02.250122Z","shell.execute_reply.started":"2024-11-04T15:57:01.68262Z","shell.execute_reply":"2024-11-04T15:57:02.249346Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile this_is_very_ugly.py\ndef this_is_very_ugly(data):\n    '''\n    It's ugly because tensorflow cannot clean the GPU memory between the training of one model to the training of the next one.\n    (or at least, I did not find such a way), so the only choice left for me was to put everything in a single huge function and run it \n    inside a multiprocess, thereby automatically cleaning the GPU memory between training runs. Well, there are probably prettier ways to structure it,\n    but it was a last-minute effort, and my priority was to get it done SOMEHOW (remember that I crossed the public LB 0.7 threshold literally a day before\n    the deadline...)\n    '''\n    from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n    from tensorflow.keras.callbacks import LearningRateScheduler, ReduceLROnPlateau\n    from tensorflow.keras import backend as K\n    import tensorflow as tf\n    print(tf.__version__)\n\n    def prepare_data_1(item):\n        '''\n        Prepare data for the ingress part: average and normalize the signal and calculate mean, std, and masks that are needed later.\n        In short, it is a preprocessing function for the model.  \n        '''\n        signal, points, jump = item\n\n        cumsum_window = 3\n        points = points-cumsum_window//2\n        cumsum = np.cumsum(signal, axis = 0)\n        window_mean = (cumsum[cumsum_window:]-cumsum[:-cumsum_window])/cumsum_window\n\n        window_mean = window_mean[1::3]\n        points = points//3\n\n        EPS_1 = 0\n        EPS_2 = 0\n        EPS_3 = 10000\n        EPS_4 = 10000\n        DEG = 3\n\n        p1, p2, p3, p4 = points\n        freq_window = 15\n\n        p1 = p1-1\n        p2 = p2+1\n        p3 = p3-1\n        p4 = p4+1\n\n        baseline_list = []\n        delta_list = []\n        err_list = []\n        fit_pols = []\n\n        X_arr = []\n        Y_arr = []\n        signal_mean_arr = []\n        signal_std_arr = []\n        x_mask_1_arr = []\n        x_mask_2_arr = []\n        for freq_idx in range(0, 282, jump):\n            min_freq = max(0, freq_idx-freq_window)\n            max_freq = min(freq_idx+freq_window+1, 282)\n            signaL_mean_freq = np.mean(window_mean[:p1, min_freq:max_freq], axis = -1)\n            signal_mean = np.mean(signaL_mean_freq)\n            signal_mean_arr.append(signal_mean)\n\n        signal_mean_arr = np.asarray(signal_mean_arr)\n        signal_std_arr = signal_mean_arr/np.min(signal_mean_arr)\n\n        signal_std_arr_real = []\n\n        for i, freq_idx in enumerate(range(0, 282, jump)):\n            min_freq = max(0, freq_idx-freq_window)\n            max_freq = min(freq_idx+freq_window+1, 282)\n            signaL_mean_freq = np.mean(window_mean[:, min_freq:max_freq], axis = -1)\n\n            signal_mean = signal_mean_arr[i]\n            signal_std = signal_std_arr[i]\n\n            signaL_mean_freq = (signaL_mean_freq-signal_mean)/signal_std\n            x = np.asarray(range(len(signaL_mean_freq)))*1.0\n            x = (x-len(x)//2)*(2/len(x))\n\n            signal_std_arr_real.append(np.std(signaL_mean_freq[:p1]))\n\n            x_mask_1 = x*0\n            x_mask_1[p2:p3+1] = 1\n            x_mask_2 = x*0+1\n            x_mask_2[p1+1:p2] = 0\n            x_mask_2[p3+1:] = 0\n\n            X_arr.append(x)\n            Y_arr.append(signaL_mean_freq)\n            x_mask_1_arr.append(x_mask_1)\n            x_mask_2_arr.append(x_mask_2)\n\n        X_arr = np.asarray(X_arr)\n        Y_arr = np.asarray(Y_arr)\n        CHANNELS = len(X_arr)\n        x_mask_1_arr = np.asarray(x_mask_1_arr)\n        x_mask_2_arr = np.asarray(x_mask_2_arr)\n        signal_std_arr_real = np.asarray(signal_std_arr_real)\n        return X_arr, Y_arr, CHANNELS, signal_mean_arr, signal_std_arr, x_mask_1_arr, x_mask_2_arr, p1, p3, signal_std_arr_real\n\n\n    def prepare_data_2(item):\n        '''\n        Prepare data for the egress part.  \n        '''\n        signal, points, jump = item\n\n        cumsum_window = 3\n        points = points-cumsum_window//2\n        cumsum = np.cumsum(signal, axis = 0)\n        window_mean = (cumsum[cumsum_window:]-cumsum[:-cumsum_window])/cumsum_window\n\n        window_mean = window_mean[1::3]\n        points = points//3\n\n        EPS_1 = 0\n        EPS_2 = 0\n        EPS_3 = 10000\n        EPS_4 = 10000\n        DEG = 3\n\n        p1, p2, p3, p4 = points\n        freq_window = 15\n\n        p1 = p1-1\n        p2 = p2+1\n        p3 = p3-1\n        p4 = p4+1\n\n        baseline_list = []\n        delta_list = []\n        err_list = []\n        fit_pols = []\n\n        X_arr = []\n        Y_arr = []\n        signal_mean_arr = []\n        signal_std_arr = []\n        x_mask_1_arr = []\n        x_mask_2_arr = []\n        for freq_idx in range(0, 282, jump):\n            min_freq = max(0, freq_idx-freq_window)\n            max_freq = min(freq_idx+freq_window+1, 282)\n            signaL_mean_freq = np.mean(window_mean[p4:, min_freq:max_freq], axis = -1)\n            signal_mean = np.mean(signaL_mean_freq)\n            signal_mean_arr.append(signal_mean)\n\n        signal_mean_arr = np.asarray(signal_mean_arr)\n        signal_std_arr = signal_mean_arr/np.min(signal_mean_arr)\n\n        signal_std_arr_real = []\n\n        for i, freq_idx in enumerate(range(0, 282, jump)):\n            min_freq = max(0, freq_idx-freq_window)\n            max_freq = min(freq_idx+freq_window+1, 282)\n            signaL_mean_freq = np.mean(window_mean[:, min_freq:max_freq], axis = -1)\n\n            signal_mean = signal_mean_arr[i]\n            signal_std = signal_std_arr[i]\n\n            signaL_mean_freq = (signaL_mean_freq-signal_mean)/signal_std\n            x = np.asarray(range(len(signaL_mean_freq)))*1.0\n            x = (x-len(x)//2)*(2/len(x))\n\n            signal_std_arr_real.append(np.std(signaL_mean_freq[p4:]))\n\n            x_mask_1 = x*0\n            x_mask_1[p2:p3+1] = 1\n            x_mask_2 = x*0+1\n            x_mask_2[:p2] = 0\n            x_mask_2[p3+1:p4] = 0\n\n            X_arr.append(x)\n            Y_arr.append(signaL_mean_freq)\n            x_mask_1_arr.append(x_mask_1)\n            x_mask_2_arr.append(x_mask_2)\n\n        X_arr = np.asarray(X_arr)\n        Y_arr = np.asarray(Y_arr)\n        CHANNELS = len(X_arr)\n        x_mask_1_arr = np.asarray(x_mask_1_arr)\n        x_mask_2_arr = np.asarray(x_mask_2_arr)\n        signal_std_arr_real = np.asarray(signal_std_arr_real)\n        return X_arr, Y_arr, CHANNELS, signal_mean_arr, signal_std_arr, x_mask_1_arr, x_mask_2_arr, p2, p4, signal_std_arr_real\n\n\n    def get_batched_data_1(items):\n        '''\n        Batch the data calculated in the preprocessing function prepare_data_1 for several planets. Here it is done for the ingress part.  \n        '''\n        x_arr = []\n        y_arr = []\n        signal_mean_arr = []\n        signal_std_arr = []\n        channels_arr = []\n        x_mask_1_arr = []\n        x_mask_2_arr = []\n        signal_std_arr_real = []\n        p1_arr = []\n        p3_arr = []\n        for item in items:\n            x, y, channels, signal_mean, signal_std, x_mask_1, x_mask_2, p1, p3, signal_std_real = prepare_data_1(item)\n            x_arr.append(x)\n            y_arr.append(y)\n            signal_mean_arr.append(signal_mean)\n            signal_std_arr.append(signal_std)\n            channels_arr.append(channels)\n            x_mask_1_arr.append(x_mask_1)\n            x_mask_2_arr.append(x_mask_2)\n            p1_arr.append(p1)\n            p3_arr.append(p3)\n            signal_std_arr_real.append(signal_std_real)\n        x_arr = np.asarray(x_arr)\n        y_arr = np.asarray(y_arr)\n        signal_mean_arr = np.asarray(signal_mean_arr)\n        signal_std_arr = np.asarray(signal_std_arr)\n        channels_arr = np.asarray(channels_arr)\n        x_mask_1_arr = np.asarray(x_mask_1_arr)\n        x_mask_2_arr = np.asarray(x_mask_2_arr)\n        p1_arr = np.asarray(p1_arr)\n        p3_arr = np.asarray(p3_arr)\n        signal_std_arr_real = np.asarray(signal_std_arr_real)\n\n        treshold = np.max(p3_arr)+1\n\n        x_arr = x_arr[:,:,:treshold]\n        y_arr = y_arr[:,:,:treshold]\n        x_mask_1_arr = x_mask_1_arr[:,:,:treshold]\n        x_mask_2_arr = x_mask_2_arr[:,:,:treshold]\n        return x_arr, y_arr, channels_arr, signal_mean_arr, signal_std_arr, x_mask_1_arr, x_mask_2_arr, p1_arr, signal_std_arr_real\n\n    def get_batched_data_2(items):\n        '''\n        Same as get_batched_data_1 but for the egress part  \n        '''\n        x_arr = []\n        y_arr = []\n        signal_mean_arr = []\n        signal_std_arr = []\n        channels_arr = []\n        x_mask_1_arr = []\n        x_mask_2_arr = []\n        signal_std_arr_real = []\n        p2_arr = []\n        p4_arr = []\n        for item in items:\n            x, y, channels, signal_mean, signal_std, x_mask_1, x_mask_2, p2, p4, signal_std_real = prepare_data_2(item)\n            x_arr.append(x)\n            y_arr.append(y)\n            signal_mean_arr.append(signal_mean)\n            signal_std_arr.append(signal_std)\n            channels_arr.append(channels)\n            x_mask_1_arr.append(x_mask_1)\n            x_mask_2_arr.append(x_mask_2)\n            p2_arr.append(p2)\n            p4_arr.append(p4)\n            signal_std_arr_real.append(signal_std_real)\n        x_arr = np.asarray(x_arr)\n        y_arr = np.asarray(y_arr)\n        signal_mean_arr = np.asarray(signal_mean_arr)\n        signal_std_arr = np.asarray(signal_std_arr)\n        channels_arr = np.asarray(channels_arr)\n        x_mask_1_arr = np.asarray(x_mask_1_arr)\n        x_mask_2_arr = np.asarray(x_mask_2_arr)\n        p2_arr = np.asarray(p2_arr)\n        p4_arr = np.asarray(p4_arr)\n        signal_std_arr_real = np.asarray(signal_std_arr_real)\n\n        treshold = np.min(p2_arr)-1\n        p4_arr = p4_arr-treshold\n\n        x_arr = x_arr[:,:,treshold:]\n        y_arr = y_arr[:,:,treshold:]\n        x_mask_1_arr = x_mask_1_arr[:,:,treshold:]\n        x_mask_2_arr = x_mask_2_arr[:,:,treshold:]\n        return x_arr, y_arr, channels_arr, signal_mean_arr, signal_std_arr, x_mask_1_arr, x_mask_2_arr, p4_arr, signal_std_arr_real\n\n    def get_final_output(x1):\n        x1 = tf.cast(x1, tf.float32)\n        return x1, x1\n\n    def get_ds():\n        '''\n        This is just a dummy dataset in order for the model to 'run' on something.  \n        '''\n        ds = tf.data.Dataset.from_tensor_slices([np.zeros((10))])\n        ds = ds.map(get_final_output, tf.data.AUTOTUNE)\n        ds = ds.repeat()\n        ds = ds.batch(1)\n        return ds\n\n    class polynom(tf.keras.layers.Layer):\n        '''\n        This layer define the 2D polynomial that is fitted against the data  \n        '''\n        def __init__(self, CHANNELS, x, batch_size, kind, **kwargs):\n            super().__init__(**kwargs)\n            self.CHANNELS = CHANNELS\n            self.x_channel = 2*(tf.range(self.CHANNELS, dtype = tf.float32)-tf.cast(\n                self.CHANNELS, tf.float32)/2)/tf.cast(self.CHANNELS, tf.float32)\n            self.x_channel = self.x_channel[None, :, None]\n            self.x_channel2 = (self.x_channel)**2\n            self.batch_size = batch_size\n            self.x = x\n            self.x2 = x**2\n            self.x3 = x**3\n            self.kind = kind\n        def build(self, input_shape):\n            self.a = self.add_weight(shape=(self.batch_size,self.CHANNELS,1), initializer=tf.keras.initializers.GlorotUniform,\n                                        regularizer = tf.keras.regularizers.l1_l2(l1 = 0.0, l2 = 0.0), dtype=tf.float32)\n            self.a_main_0 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.a_main_1 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.a_main_2 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n\n            self.b_main_0 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.b_main_1 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.b_main_2 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n\n            self.c_main_0 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.c_main_1 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.c_main_2 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n\n            self.d_main_0 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.d_main_1 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n            self.d_main_2 = self.add_weight(shape=(self.batch_size,1,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n        def call(self, x):\n            if self.kind == 1:\n                a_main = self.a_main_0+self.x_channel*self.a_main_1+self.x_channel2*self.a_main_2\n                b_main = self.b_main_0+self.x_channel*self.b_main_1\n                c_main = self.c_main_0+self.x_channel*self.c_main_1\n                d_main = self.d_main_0\n            else:\n                a_main = self.a_main_0+self.x_channel*self.a_main_1+self.x_channel2*self.a_main_2\n                b_main = self.b_main_0+self.x_channel*self.b_main_1+self.x_channel2*self.b_main_2\n                c_main = self.c_main_0+self.x_channel*self.c_main_1+self.x_channel2*self.c_main_2\n                d_main = self.d_main_0+self.x_channel*self.d_main_1\n            if self.kind == 1:\n                x = (a_main+self.a)+(b_main)*self.x+(c_main)*self.x2\n            else:\n                x = (a_main+self.a)+(b_main)*self.x+(c_main)*self.x2+0.1*(d_main)*self.x3\n            return x\n\n    class layer_1(tf.keras.layers.Layer):\n        '''\n        This layer is the heart of the model- this is the logic. It defines the polynomial and the 'x_delta' variable, that is, the difference in the signal\n        between the ingress and egress compared to outside, and performs the necessary masking and calculations.  \n        mask_1 is there to mask the signal that is outside the ingress-egress section.\n        mask_2 mask the sections that are between the start/end of the ingress and the start/end of the egress.\n        mask_3 normalizes the results by the std per wavelength (well, it's not a mask, but the name remains for historical reasons...) \n        '''\n        def __init__(self, x_arr, y_arr, x_mask_1_arr, x_mask_2_arr, CHANNELS, x_batch_size, x_mask_3_arr, kind, **kwargs):\n            super().__init__(**kwargs)\n            self.CHANNELS = CHANNELS\n            self.x = x_arr\n            self.y = y_arr\n            self.x_mask_1 = x_mask_1_arr\n            self.x_mask_2 = x_mask_2_arr\n            self.batch_size = x_batch_size\n            self.x_mask_3 = x_mask_3_arr\n            self.kind = kind\n        def build(self, input_shape):\n            self.polynom_layer = polynom(self.CHANNELS, self.x, self.batch_size, self.kind)\n            self.DELTA = self.add_weight(shape=(self.batch_size,self.CHANNELS,1), initializer=tf.keras.initializers.GlorotUniform, dtype=tf.float32)\n        def call(self, x):\n            x = self.polynom_layer(x)\n            x_delta = ((0.0*self.x)+self.DELTA)*self.x_mask_1\n            x = x+x_delta\n            x = ((x-self.y)*self.x_mask_2)/self.x_mask_3\n            return x\n\n    def loss_fn(x, preds):\n        '''\n        The part of 'pred-target' was already performed in 'layer_1' since it was more natural at the time, so\n        here, all that is left is to calculate the mean square.\n        '''\n        loss = (tf.math.abs(preds))**2\n        return tf.reduce_mean(loss)\n\n    def get_model(x_arr, y_arr, x_mask_1_arr, x_mask_2_arr, CHANNELS, x_batch_size, x_mask_3_arr, kind):\n        x_arr = tf.cast(tf.convert_to_tensor(x_arr), tf.float32)\n        y_arr = tf.cast(tf.convert_to_tensor(y_arr), tf.float32)\n\n        inp1 = tf.keras.Input([10])\n\n        x = inp1\n\n        x = layer_1(x_arr, y_arr, x_mask_1_arr, x_mask_2_arr, CHANNELS, x_batch_size, x_mask_3_arr, kind)(x)\n\n        model = tf.keras.Model(inp1, x)\n\n        optimizer = tf.keras.optimizers.AdamW(learning_rate=0.0005)\n        loss = loss_fn\n        model.compile(loss=loss, optimizer=optimizer)\n        return model\n\n\n    N_EPOCHS = 20\n    N_WARMUP_EPOCHS = 0\n    LR_MAX = 7e-3\n    WD_RATIO = 4.0\n\n    def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n        if current_step < num_warmup_steps:\n            if WARMUP_METHOD == 'log':\n                return lr_max * 0.10 ** (num_warmup_steps - current_step)\n            else:\n                return lr_max * 2 ** -(num_warmup_steps - current_step)\n        else:\n            progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n            return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max\n\n    # Learning rate for encoder\n    LR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50, num_training_steps = N_EPOCHS)\n        for step in range(N_EPOCHS)]\n    #LR_SCHEDULE = list(np.linspace(LR_MAX, LR_MIN, num=N_EPOCHS))\n    # Learning Rate Callback\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=0)\n\n\n    def regression_function(items):\n        '''\n        This function combines the preprocessing, modeling, and training, denormalizes the output, and calculate the final predictions  \n        '''\n        # Custom callback to update weight decay with learning rate\n        class WeightDecayCallback(tf.keras.callbacks.Callback):\n            def __init__(self, wd_ratio=WD_RATIO):\n                self.step_counter = 0\n                self.wd_ratio = wd_ratio\n\n            def on_epoch_begin(self, epoch, logs=None):\n                model.optimizer.weight_decay = model.optimizer.learning_rate * self.wd_ratio\n                print(f'learning rate: {model.optimizer.learning_rate.numpy():.2e}, weight decay: {model.optimizer.weight_decay.numpy():.2e}')\n\n        items, idx, kind, save_folder_path = items\n        if kind == 1:\n            x_arr, y_arr, channels_arr, signal_mean_arr, signal_std_arr, x_mask_1_arr, x_mask_2_arr, p1_arr, signal_std_arr_real = get_batched_data_1(items)\n        else:\n            x_arr, y_arr, channels_arr, signal_mean_arr, signal_std_arr, x_mask_1_arr, x_mask_2_arr, p1_arr, signal_std_arr_real = get_batched_data_2(items)\n        CHANNELS = channels_arr[0]\n        x_batch_size = len(x_arr)\n\n        x_mask_3_arr = x_mask_1_arr*0+signal_std_arr_real[:,:,None]/np.max(signal_std_arr_real, axis = 1)[:, None, None]\n\n        ds = get_ds()\n        laa = [x for x in ds.take(1)]\n        X = laa[0][0]\n\n        model = get_model(x_arr, y_arr, x_mask_1_arr, x_mask_2_arr, CHANNELS, x_batch_size, x_mask_3_arr, kind)\n        history = model.fit(ds, verbose=2,\n                            steps_per_epoch = 500,\n                            epochs=N_EPOCHS, batch_size=1,\n                            callbacks=[lr_callback, WeightDecayCallback()])\n\n        DELTA = model.layers[1].DELTA.value.numpy()\n        DELTA = DELTA[:,:,0]\n        DELTA = -DELTA*signal_std_arr\n\n        baseline = np.concatenate([(model.layers[1].polynom_layer(X))[i:i+1, :, p1_arr[i]] for i in range(len(p1_arr))], axis = 0)\n        baseline = baseline*signal_std_arr+signal_mean_arr\n\n        relred = ((DELTA/(baseline))[:, ::-1])\n        pickle.dump(relred, open(f'{save_folder_path}/relred_{kind}_{idx}.p', 'bw'))\n\n    regression_function(data)","metadata":{"execution":{"iopub.status.busy":"2024-11-04T15:57:02.251768Z","iopub.execute_input":"2024-11-04T15:57:02.2524Z","iopub.status.idle":"2024-11-04T15:57:02.271214Z","shell.execute_reply.started":"2024-11-04T15:57:02.252366Z","shell.execute_reply":"2024-11-04T15:57:02.270345Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile get_train.py\ndef get_train(a_raw_train, points_list, DEBUG = False):\n    '''\n    This function runs the training inside a multiprocess, gets the results\n    '''\n    folder = 108\n    save_folder_path = f'/kaggle/working/{folder}'\n    try:\n        os.mkdir(save_folder_path)\n    except:\n        pass\n    \n    \n    batch_size = 256\n    end = len(points_list)\n    strides = 5\n    if DEBUG:\n        batch_size = 2\n        end = 2\n        strides = 5\n    for idx, start in enumerate(range(0, end, batch_size)):\n        items = [[[a_raw_train[i], points_list[i], strides] for i in range(start, min(start+batch_size, len(points_list)))], idx, 1, save_folder_path]\n\n        process_eval = multiprocessing.Process(target=this_is_very_ugly, args=([items]))\n        process_eval.start()\n        process_eval.join()\n    for idx, start in enumerate(range(0, end, batch_size)):\n        items = [[[a_raw_train[i], points_list[i], strides] for i in range(start, min(start+batch_size, len(points_list)))], idx, 2, save_folder_path]\n\n        process_eval = multiprocessing.Process(target=this_is_very_ugly, args=([items]))\n        process_eval.start()\n        process_eval.join()\n\n    new_train_data = [pickle.load(open(f'{save_folder_path}/relred_1_{x}.p', 'br')) for x in range(len(os.listdir(f'{save_folder_path}'))//2)]\n    new_train_data = np.concatenate(new_train_data, axis = 0)\n    new_train_data_1 = np.repeat(new_train_data, strides, axis = 1)[:, strides//2:strides//2+282]\n\n    new_train_data = [pickle.load(open(f'{save_folder_path}/relred_2_{x}.p', 'br')) for x in range(len(os.listdir(f'{save_folder_path}'))//2)]\n    new_train_data = np.concatenate(new_train_data, axis = 0)\n    new_train_data_2 = np.repeat(new_train_data, strides, axis = 1)[:, strides//2:strides//2+282]\n\n    return new_train_data_1, new_train_data_2","metadata":{"execution":{"iopub.status.busy":"2024-11-04T15:57:02.272439Z","iopub.execute_input":"2024-11-04T15:57:02.272782Z","iopub.status.idle":"2024-11-04T15:57:02.286595Z","shell.execute_reply.started":"2024-11-04T15:57:02.272748Z","shell.execute_reply":"2024-11-04T15:57:02.285263Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"a_raw_train = pickle.load(open('/kaggle/input/adc24-preprocessing/a_raw_train.pickle', 'br'))\ntrain_labels = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_labels.csv',\n                        index_col='planet_id').to_numpy()[:, 1:]\nstar = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv').star.to_numpy()\npoints_list = pickle.load(open('/kaggle/input/adc24-preprocessing/points_list.p', 'br'))\nadvanced_fitting_results = pickle.load(open('/kaggle/input/adc24-preprocessing/advanced_fitting_results.p', 'br'))\ntrain_adc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/train_adc_info.csv',\n                           index_col='planet_id')\nwavelengths = pd.read_csv('/kaggle/input/ariel-data-challenge-2024/wavelengths.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-04T15:57:02.288903Z","iopub.execute_input":"2024-11-04T15:57:02.289576Z","iopub.status.idle":"2024-11-04T15:57:43.977885Z","shell.execute_reply.started":"2024-11-04T15:57:02.289539Z","shell.execute_reply":"2024-11-04T15:57:43.977036Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exec(open('this_is_very_ugly.py', 'r').read())\nexec(open('get_train.py', 'r').read())","metadata":{"execution":{"iopub.status.busy":"2024-11-04T15:57:43.979274Z","iopub.execute_input":"2024-11-04T15:57:43.980007Z","iopub.status.idle":"2024-11-04T15:57:43.991142Z","shell.execute_reply.started":"2024-11-04T15:57:43.97996Z","shell.execute_reply":"2024-11-04T15:57:43.990249Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train_data_1, new_train_data_2 = get_train(a_raw_train, points_list, DEBUG = False)\npickle.dump(new_train_data_1, open('new_train_data_1.p', 'bw'))\npickle.dump(new_train_data_2, open('new_train_data_2.p', 'bw'))","metadata":{"execution":{"iopub.status.busy":"2024-11-04T15:57:43.993561Z","iopub.execute_input":"2024-11-04T15:57:43.994021Z","iopub.status.idle":"2024-11-04T15:59:25.36696Z","shell.execute_reply.started":"2024-11-04T15:57:43.993987Z","shell.execute_reply":"2024-11-04T15:59:25.365613Z"},"trusted":true},"outputs":[],"execution_count":null}]}