{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"import os","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:41:15.589409Z","iopub.execute_input":"2024-04-22T08:41:15.589947Z","iopub.status.idle":"2024-04-22T08:41:15.595788Z","shell.execute_reply.started":"2024-04-22T08:41:15.589911Z","shell.execute_reply":"2024-04-22T08:41:15.594555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_INTERACTIVE = (os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '') == 'Interactive')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:41:37.277689Z","iopub.execute_input":"2024-04-22T08:41:37.278101Z","iopub.status.idle":"2024-04-22T08:41:37.285019Z","shell.execute_reply.started":"2024-04-22T08:41:37.278072Z","shell.execute_reply":"2024-04-22T08:41:37.283556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"import polars as pl","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:33:48.70962Z","iopub.execute_input":"2024-04-22T07:33:48.710877Z","iopub.status.idle":"2024-04-22T07:33:49.125839Z","shell.execute_reply.started":"2024-04-22T07:33:48.710793Z","shell.execute_reply":"2024-04-22T07:33:49.124374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset is huge. Here we only look at the first 10000 samples for interactive session and 100000 samples for batch session.","metadata":{}},{"cell_type":"code","source":"NUM_TRAIN_SAMPLES = 10_000 if IS_INTERACTIVE else 100_000","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:42:19.270299Z","iopub.execute_input":"2024-04-22T08:42:19.270699Z","iopub.status.idle":"2024-04-22T08:42:19.27637Z","shell.execute_reply.started":"2024-04-22T08:42:19.27067Z","shell.execute_reply":"2024-04-22T08:42:19.274912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lf_train = pl.scan_csv(\n    '/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv',\n).head(NUM_TRAIN_SAMPLES).drop('sample_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:34:38.459037Z","iopub.execute_input":"2024-04-22T07:34:38.459416Z","iopub.status.idle":"2024-04-22T07:34:38.483556Z","shell.execute_reply.started":"2024-04-22T07:34:38.459387Z","shell.execute_reply":"2024-04-22T07:34:38.482171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.head().collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:34:40.725274Z","iopub.execute_input":"2024-04-22T07:34:40.725673Z","iopub.status.idle":"2024-04-22T07:34:40.778179Z","shell.execute_reply.started":"2024-04-22T07:34:40.725646Z","shell.execute_reply":"2024-04-22T07:34:40.776885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic description","metadata":{}},{"cell_type":"code","source":"df_schema = pl.DataFrame(\n    list(lf_train.schema.items()),\n    schema = [('feature', pl.String), ('dtype', pl.Object)],\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:33:53.971265Z","iopub.execute_input":"2024-04-22T07:33:53.971727Z","iopub.status.idle":"2024-04-22T07:33:53.999434Z","shell.execute_reply.started":"2024-04-22T07:33:53.971694Z","shell.execute_reply":"2024-04-22T07:33:53.998248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_schema)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:33:54.729572Z","iopub.execute_input":"2024-04-22T07:33:54.730057Z","iopub.status.idle":"2024-04-22T07:33:54.740306Z","shell.execute_reply.started":"2024-04-22T07:33:54.730021Z","shell.execute_reply":"2024-04-22T07:33:54.738733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_schema.select('dtype').to_series().value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:33:55.043668Z","iopub.execute_input":"2024-04-22T07:33:55.044158Z","iopub.status.idle":"2024-04-22T07:33:55.05892Z","shell.execute_reply.started":"2024-04-22T07:33:55.044112Z","shell.execute_reply":"2024-04-22T07:33:55.057909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEATURE_GROUPS_60 = [\n    'state_t',\n    'state_q0001',\n    'state_q0002',\n    'state_q0003',\n    'state_u',\n    'state_v',\n    'pbuf_ozone',\n    'pbuf_CH4',\n    'pbuf_N2O',\n]\nFEATURE_GROUPS_1 = [\n    'state_ps',\n    'pbuf_SOLIN',\n    'pbuf_LHFLX',\n    'pbuf_SHFLX',\n    'pbuf_TAUX',\n    'pbuf_TAUY',\n    'pbuf_COSZRS',\n    'cam_in_ALDIF',\n    'cam_in_ALDIR',\n    'cam_in_ASDIF',\n    'cam_in_ASDIR',\n    'cam_in_LWUP',\n    'cam_in_ICEFRAC',\n    'cam_in_LANDFRAC',\n    'cam_in_OCNFRAC',\n    'cam_in_SNOWHLAND',\n]\nTARGET_GROUPS_60 = [\n    'ptend_t',\n    'ptend_q0001',\n    'ptend_q0002',\n    'ptend_q0003',\n    'ptend_u',\n    'ptend_v',\n]\nTARGET_GROUPS_1 = [\n    'cam_out_NETSW',\n    'cam_out_FLWDS',\n    'cam_out_PRECSC',\n    'cam_out_PRECC',\n    'cam_out_SOLS',\n    'cam_out_SOLL',\n    'cam_out_SOLSD',\n    'cam_out_SOLLD',\n]","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:16:26.4714Z","iopub.execute_input":"2024-04-22T08:16:26.471797Z","iopub.status.idle":"2024-04-22T08:16:26.480042Z","shell.execute_reply.started":"2024-04-22T08:16:26.471758Z","shell.execute_reply":"2024-04-22T08:16:26.478648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(pl.concat_list(pl.all().null_count()).list.sum().alias('total_null_count')).collect())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:33:55.605147Z","iopub.execute_input":"2024-04-22T07:33:55.606095Z","iopub.status.idle":"2024-04-22T07:33:56.683781Z","shell.execute_reply.started":"2024-04-22T07:33:55.606054Z","shell.execute_reply":"2024-04-22T07:33:56.682175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- All features and targets are float.\n- No missing values.","metadata":{}},{"cell_type":"markdown","source":"# Exploration 1","metadata":{}},{"cell_type":"markdown","source":"- Statistics and histogram+KDE for every feature/target.\n- Correlation between features/targets in every group.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport polars.selectors as cs\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:48:55.003632Z","iopub.execute_input":"2024-04-22T07:48:55.004081Z","iopub.status.idle":"2024-04-22T07:48:55.011257Z","shell.execute_reply.started":"2024-04-22T07:48:55.004049Z","shell.execute_reply":"2024-04-22T07:48:55.009716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('state_t')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:44:28.395304Z","iopub.execute_input":"2024-04-22T07:44:28.39578Z","iopub.status.idle":"2024-04-22T07:44:28.674153Z","shell.execute_reply.started":"2024-04-22T07:44:28.395745Z","shell.execute_reply":"2024-04-22T07:44:28.672906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_feature_60(lf, feature):\n    _, axes = plt.subplots(10, 6, figsize = (30, 36), layout = 'tight')\n    with warnings.catch_warnings():\n        warnings.simplefilter('ignore')\n        for i in range(10):\n            for j in range(6):\n                n = 6 * i + j\n                sns.histplot(x = lf_train.select(f'{feature}_{n}').collect().to_series().to_pandas(),\n                             ax = axes[i, j],\n                             bins = 'sturges',\n                             kde = True)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:59:29.676404Z","iopub.execute_input":"2024-04-22T07:59:29.676904Z","iopub.status.idle":"2024-04-22T07:59:29.687344Z","shell.execute_reply.started":"2024-04-22T07:59:29.676867Z","shell.execute_reply":"2024-04-22T07:59:29.685774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'state_t')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T07:59:37.620098Z","iopub.execute_input":"2024-04-22T07:59:37.620556Z","iopub.status.idle":"2024-04-22T08:00:14.62707Z","shell.execute_reply.started":"2024-04-22T07:59:37.620522Z","shell.execute_reply":"2024-04-22T08:00:14.625679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_corr_60(lf, feature):\n    _, ax = plt.subplots(figsize = (36, 32))\n    sns.heatmap(lf_train.select(cs.starts_with(feature)).collect().to_pandas().corr(),\n                ax = ax,\n                vmin = -1, vmax = 1, cmap = 'coolwarm', annot = True)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:00:22.3325Z","iopub.execute_input":"2024-04-22T08:00:22.332968Z","iopub.status.idle":"2024-04-22T08:00:22.340472Z","shell.execute_reply.started":"2024-04-22T08:00:22.332935Z","shell.execute_reply":"2024-04-22T08:00:22.339008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'state_t')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:00:29.428444Z","iopub.execute_input":"2024-04-22T08:00:29.428921Z","iopub.status.idle":"2024-04-22T08:00:42.053904Z","shell.execute_reply.started":"2024-04-22T08:00:29.428883Z","shell.execute_reply":"2024-04-22T08:00:42.050644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('state_q0001')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:06:07.877069Z","iopub.execute_input":"2024-04-22T08:06:07.877574Z","iopub.status.idle":"2024-04-22T08:06:08.08823Z","shell.execute_reply.started":"2024-04-22T08:06:07.877526Z","shell.execute_reply":"2024-04-22T08:06:08.087349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'state_q0001')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:01:06.577106Z","iopub.execute_input":"2024-04-22T08:01:06.578166Z","iopub.status.idle":"2024-04-22T08:01:44.351129Z","shell.execute_reply.started":"2024-04-22T08:01:06.578108Z","shell.execute_reply":"2024-04-22T08:01:44.349545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'state_q0001')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:02:25.886158Z","iopub.execute_input":"2024-04-22T08:02:25.886629Z","iopub.status.idle":"2024-04-22T08:02:38.711979Z","shell.execute_reply.started":"2024-04-22T08:02:25.886593Z","shell.execute_reply":"2024-04-22T08:02:38.710503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('state_q0002')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:06:40.183408Z","iopub.execute_input":"2024-04-22T08:06:40.184108Z","iopub.status.idle":"2024-04-22T08:06:40.37795Z","shell.execute_reply.started":"2024-04-22T08:06:40.184072Z","shell.execute_reply":"2024-04-22T08:06:40.376779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'state_q0002')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:03:55.414563Z","iopub.execute_input":"2024-04-22T08:03:55.415096Z","iopub.status.idle":"2024-04-22T08:04:33.354286Z","shell.execute_reply.started":"2024-04-22T08:03:55.415039Z","shell.execute_reply":"2024-04-22T08:04:33.352917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'state_q0002')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:04:33.356538Z","iopub.execute_input":"2024-04-22T08:04:33.357014Z","iopub.status.idle":"2024-04-22T08:04:46.118342Z","shell.execute_reply.started":"2024-04-22T08:04:33.356978Z","shell.execute_reply":"2024-04-22T08:04:46.116549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('state_q0003')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:07:48.518709Z","iopub.execute_input":"2024-04-22T08:07:48.519147Z","iopub.status.idle":"2024-04-22T08:07:48.712358Z","shell.execute_reply.started":"2024-04-22T08:07:48.519115Z","shell.execute_reply":"2024-04-22T08:07:48.711519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'state_q0003')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:08:09.423731Z","iopub.execute_input":"2024-04-22T08:08:09.424731Z","iopub.status.idle":"2024-04-22T08:08:46.433389Z","shell.execute_reply.started":"2024-04-22T08:08:09.424696Z","shell.execute_reply":"2024-04-22T08:08:46.431634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'state_q0003')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:08:46.435944Z","iopub.execute_input":"2024-04-22T08:08:46.436443Z","iopub.status.idle":"2024-04-22T08:08:59.397146Z","shell.execute_reply.started":"2024-04-22T08:08:46.436401Z","shell.execute_reply":"2024-04-22T08:08:59.393135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('state_u')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:10:18.148123Z","iopub.execute_input":"2024-04-22T08:10:18.148609Z","iopub.status.idle":"2024-04-22T08:10:18.352956Z","shell.execute_reply.started":"2024-04-22T08:10:18.148571Z","shell.execute_reply":"2024-04-22T08:10:18.35203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'state_u')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:10:37.598373Z","iopub.execute_input":"2024-04-22T08:10:37.599095Z","iopub.status.idle":"2024-04-22T08:11:14.111869Z","shell.execute_reply.started":"2024-04-22T08:10:37.59906Z","shell.execute_reply":"2024-04-22T08:11:14.108102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'state_u')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:11:14.114121Z","iopub.execute_input":"2024-04-22T08:11:14.114486Z","iopub.status.idle":"2024-04-22T08:11:26.490383Z","shell.execute_reply.started":"2024-04-22T08:11:14.114457Z","shell.execute_reply":"2024-04-22T08:11:26.486777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('state_v')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:12:41.521348Z","iopub.execute_input":"2024-04-22T08:12:41.521878Z","iopub.status.idle":"2024-04-22T08:12:41.721721Z","shell.execute_reply.started":"2024-04-22T08:12:41.521828Z","shell.execute_reply":"2024-04-22T08:12:41.720786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'state_v')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:12:57.643883Z","iopub.execute_input":"2024-04-22T08:12:57.645031Z","iopub.status.idle":"2024-04-22T08:13:35.221223Z","shell.execute_reply.started":"2024-04-22T08:12:57.644994Z","shell.execute_reply":"2024-04-22T08:13:35.219696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'state_v')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:13:35.224123Z","iopub.execute_input":"2024-04-22T08:13:35.225875Z","iopub.status.idle":"2024-04-22T08:13:47.793061Z","shell.execute_reply.started":"2024-04-22T08:13:35.225797Z","shell.execute_reply":"2024-04-22T08:13:47.78979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('pbuf_ozone')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:14:59.800541Z","iopub.execute_input":"2024-04-22T08:14:59.801442Z","iopub.status.idle":"2024-04-22T08:15:00.006393Z","shell.execute_reply.started":"2024-04-22T08:14:59.80139Z","shell.execute_reply":"2024-04-22T08:15:00.005305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'pbuf_ozone')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:15:21.301638Z","iopub.execute_input":"2024-04-22T08:15:21.302147Z","iopub.status.idle":"2024-04-22T08:15:58.305401Z","shell.execute_reply.started":"2024-04-22T08:15:21.302113Z","shell.execute_reply":"2024-04-22T08:15:58.304039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'pbuf_ozone')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:15:58.307545Z","iopub.execute_input":"2024-04-22T08:15:58.308003Z","iopub.status.idle":"2024-04-22T08:16:10.498708Z","shell.execute_reply.started":"2024-04-22T08:15:58.307969Z","shell.execute_reply":"2024-04-22T08:16:10.496861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('pbuf_CH4')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:17:05.387529Z","iopub.execute_input":"2024-04-22T08:17:05.388001Z","iopub.status.idle":"2024-04-22T08:17:05.566786Z","shell.execute_reply.started":"2024-04-22T08:17:05.387969Z","shell.execute_reply":"2024-04-22T08:17:05.565203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'pbuf_CH4')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:17:22.926474Z","iopub.execute_input":"2024-04-22T08:17:22.926919Z","iopub.status.idle":"2024-04-22T08:17:57.371687Z","shell.execute_reply.started":"2024-04-22T08:17:22.926886Z","shell.execute_reply":"2024-04-22T08:17:57.37063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'pbuf_CH4')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:17:57.373713Z","iopub.execute_input":"2024-04-22T08:17:57.374763Z","iopub.status.idle":"2024-04-22T08:18:01.615406Z","shell.execute_reply.started":"2024-04-22T08:17:57.374721Z","shell.execute_reply":"2024-04-22T08:18:01.613661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('pbuf_N2O')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:18:53.116129Z","iopub.execute_input":"2024-04-22T08:18:53.116658Z","iopub.status.idle":"2024-04-22T08:18:53.311473Z","shell.execute_reply.started":"2024-04-22T08:18:53.11662Z","shell.execute_reply":"2024-04-22T08:18:53.310489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_60(lf_train, 'pbuf_N2O')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:19:18.422271Z","iopub.execute_input":"2024-04-22T08:19:18.423596Z","iopub.status.idle":"2024-04-22T08:19:48.838539Z","shell.execute_reply.started":"2024-04-22T08:19:18.423541Z","shell.execute_reply":"2024-04-22T08:19:48.837372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'pbuf_N2O')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:19:48.840745Z","iopub.execute_input":"2024-04-22T08:19:48.841521Z","iopub.status.idle":"2024-04-22T08:19:53.80337Z","shell.execute_reply.started":"2024-04-22T08:19:48.841479Z","shell.execute_reply":"2024-04-22T08:19:53.801906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(FEATURE_GROUPS_1).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:24:16.466907Z","iopub.execute_input":"2024-04-22T08:24:16.468346Z","iopub.status.idle":"2024-04-22T08:24:16.624292Z","shell.execute_reply.started":"2024-04-22T08:24:16.468294Z","shell.execute_reply":"2024-04-22T08:24:16.622723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = len(FEATURE_GROUPS_1)\n_, axes = plt.subplots(n, figsize = (6, 3 * n), layout = 'tight')\nwith warnings.catch_warnings():\n    warnings.simplefilter('ignore')\n    for ax, c in zip(axes, FEATURE_GROUPS_1):\n        sns.histplot(x = lf_train.select(c).collect().to_series().to_pandas(),\n                     ax = ax,\n                     bins = 'sturges',\n                     kde = True)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:21:56.441325Z","iopub.execute_input":"2024-04-22T08:21:56.442474Z","iopub.status.idle":"2024-04-22T08:22:05.743464Z","shell.execute_reply.started":"2024-04-22T08:21:56.442426Z","shell.execute_reply":"2024-04-22T08:22:05.742044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('ptend_t')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:24:02.851735Z","iopub.execute_input":"2024-04-22T08:24:02.853024Z","iopub.status.idle":"2024-04-22T08:24:03.035675Z","shell.execute_reply.started":"2024-04-22T08:24:02.852979Z","shell.execute_reply":"2024-04-22T08:24:03.034826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60 = plot_feature_60","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:25:26.118374Z","iopub.execute_input":"2024-04-22T08:25:26.11897Z","iopub.status.idle":"2024-04-22T08:25:26.125565Z","shell.execute_reply.started":"2024-04-22T08:25:26.11891Z","shell.execute_reply":"2024-04-22T08:25:26.123961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60(lf_train, 'ptend_t')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:25:28.799268Z","iopub.execute_input":"2024-04-22T08:25:28.799674Z","iopub.status.idle":"2024-04-22T08:26:05.714409Z","shell.execute_reply.started":"2024-04-22T08:25:28.799644Z","shell.execute_reply":"2024-04-22T08:26:05.713195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'ptend_t')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:26:05.716433Z","iopub.execute_input":"2024-04-22T08:26:05.717733Z","iopub.status.idle":"2024-04-22T08:26:18.863279Z","shell.execute_reply.started":"2024-04-22T08:26:05.717684Z","shell.execute_reply":"2024-04-22T08:26:18.861594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('ptend_q0001')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:27:00.020197Z","iopub.execute_input":"2024-04-22T08:27:00.021117Z","iopub.status.idle":"2024-04-22T08:27:00.208155Z","shell.execute_reply.started":"2024-04-22T08:27:00.021074Z","shell.execute_reply":"2024-04-22T08:27:00.207078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60(lf_train, 'ptend_q0001')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:27:26.312898Z","iopub.execute_input":"2024-04-22T08:27:26.313663Z","iopub.status.idle":"2024-04-22T08:28:00.024203Z","shell.execute_reply.started":"2024-04-22T08:27:26.313618Z","shell.execute_reply":"2024-04-22T08:28:00.022761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'ptend_q0001')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:28:00.026707Z","iopub.execute_input":"2024-04-22T08:28:00.027214Z","iopub.status.idle":"2024-04-22T08:28:10.26463Z","shell.execute_reply.started":"2024-04-22T08:28:00.027177Z","shell.execute_reply":"2024-04-22T08:28:10.263246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('ptend_q0002')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:28:46.234917Z","iopub.execute_input":"2024-04-22T08:28:46.235359Z","iopub.status.idle":"2024-04-22T08:28:46.410687Z","shell.execute_reply.started":"2024-04-22T08:28:46.235327Z","shell.execute_reply":"2024-04-22T08:28:46.40978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60(lf_train, 'ptend_q0002')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:29:08.689305Z","iopub.execute_input":"2024-04-22T08:29:08.690183Z","iopub.status.idle":"2024-04-22T08:29:43.233891Z","shell.execute_reply.started":"2024-04-22T08:29:08.690147Z","shell.execute_reply":"2024-04-22T08:29:43.232369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'ptend_q0002')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:29:43.236254Z","iopub.execute_input":"2024-04-22T08:29:43.236589Z","iopub.status.idle":"2024-04-22T08:29:52.205059Z","shell.execute_reply.started":"2024-04-22T08:29:43.236559Z","shell.execute_reply":"2024-04-22T08:29:52.203568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('ptend_q0003')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:30:30.193129Z","iopub.execute_input":"2024-04-22T08:30:30.19358Z","iopub.status.idle":"2024-04-22T08:30:30.374718Z","shell.execute_reply.started":"2024-04-22T08:30:30.193546Z","shell.execute_reply":"2024-04-22T08:30:30.373616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60(lf_train, 'ptend_q0003')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:30:43.872329Z","iopub.execute_input":"2024-04-22T08:30:43.872979Z","iopub.status.idle":"2024-04-22T08:31:18.085303Z","shell.execute_reply.started":"2024-04-22T08:30:43.872945Z","shell.execute_reply":"2024-04-22T08:31:18.083721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'ptend_q0003')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:31:18.087588Z","iopub.execute_input":"2024-04-22T08:31:18.088041Z","iopub.status.idle":"2024-04-22T08:31:27.630027Z","shell.execute_reply.started":"2024-04-22T08:31:18.088003Z","shell.execute_reply":"2024-04-22T08:31:27.628214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('ptend_u')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:33:11.769827Z","iopub.execute_input":"2024-04-22T08:33:11.770337Z","iopub.status.idle":"2024-04-22T08:33:11.949986Z","shell.execute_reply.started":"2024-04-22T08:33:11.770301Z","shell.execute_reply":"2024-04-22T08:33:11.94905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60(lf_train, 'ptend_u')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:33:31.874614Z","iopub.execute_input":"2024-04-22T08:33:31.876211Z","iopub.status.idle":"2024-04-22T08:34:06.124374Z","shell.execute_reply.started":"2024-04-22T08:33:31.876164Z","shell.execute_reply":"2024-04-22T08:34:06.123061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'ptend_u')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:34:06.126788Z","iopub.execute_input":"2024-04-22T08:34:06.127206Z","iopub.status.idle":"2024-04-22T08:34:15.44654Z","shell.execute_reply.started":"2024-04-22T08:34:06.127171Z","shell.execute_reply":"2024-04-22T08:34:15.445096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(cs.starts_with('ptend_v')).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:34:15.448464Z","iopub.execute_input":"2024-04-22T08:34:15.44895Z","iopub.status.idle":"2024-04-22T08:34:15.626958Z","shell.execute_reply.started":"2024-04-22T08:34:15.448909Z","shell.execute_reply":"2024-04-22T08:34:15.625639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_target_60(lf_train, 'ptend_v')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:34:15.629799Z","iopub.execute_input":"2024-04-22T08:34:15.631135Z","iopub.status.idle":"2024-04-22T08:34:48.947569Z","shell.execute_reply.started":"2024-04-22T08:34:15.631082Z","shell.execute_reply":"2024-04-22T08:34:48.945954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_corr_60(lf_train, 'ptend_v')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:34:48.949386Z","iopub.execute_input":"2024-04-22T08:34:48.949838Z","iopub.status.idle":"2024-04-22T08:34:57.920354Z","shell.execute_reply.started":"2024-04-22T08:34:48.949779Z","shell.execute_reply":"2024-04-22T08:34:57.918464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(lf_train.select(TARGET_GROUPS_1).describe())","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:35:24.516216Z","iopub.execute_input":"2024-04-22T08:35:24.516621Z","iopub.status.idle":"2024-04-22T08:35:24.629642Z","shell.execute_reply.started":"2024-04-22T08:35:24.516591Z","shell.execute_reply":"2024-04-22T08:35:24.628771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = len(TARGET_GROUPS_1)\n_, axes = plt.subplots(n, figsize = (6, 3 * n), layout = 'tight')\nwith warnings.catch_warnings():\n    warnings.simplefilter('ignore')\n    for ax, c in zip(axes, TARGET_GROUPS_1):\n        sns.histplot(x = lf_train.select(c).collect().to_series().to_pandas(),\n                     ax = ax,\n                     bins = 'sturges', kde = True)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:36:32.581832Z","iopub.execute_input":"2024-04-22T08:36:32.582287Z","iopub.status.idle":"2024-04-22T08:36:38.207105Z","shell.execute_reply.started":"2024-04-22T08:36:32.582256Z","shell.execute_reply":"2024-04-22T08:36:38.205436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}