{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import glob\nimport numpy as np\nimport pandas as pd \nimport os\n\nimport h5py\n\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-12T16:00:57.957519Z","iopub.execute_input":"2022-10-12T16:00:57.958058Z","iopub.status.idle":"2022-10-12T16:00:58.137424Z","shell.execute_reply.started":"2022-10-12T16:00:57.957992Z","shell.execute_reply":"2022-10-12T16:00:58.136088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2022-10-12T15:57:38.28764Z","iopub.execute_input":"2022-10-12T15:57:38.28838Z","iopub.status.idle":"2022-10-12T15:57:38.293987Z","shell.execute_reply.started":"2022-10-12T15:57:38.28834Z","shell.execute_reply":"2022-10-12T15:57:38.292822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_labels = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/train_labels.csv')\nprint(train_labels.shape)\ntrain_labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-12T15:57:38.538626Z","iopub.execute_input":"2022-10-12T15:57:38.539108Z","iopub.status.idle":"2022-10-12T15:57:38.558995Z","shell.execute_reply.started":"2022-10-12T15:57:38.539068Z","shell.execute_reply":"2022-10-12T15:57:38.558099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nss = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\nprint(ss.shape)\nss.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-12T15:57:38.800828Z","iopub.execute_input":"2022-10-12T15:57:38.802196Z","iopub.status.idle":"2022-10-12T15:57:38.823195Z","shell.execute_reply.started":"2022-10-12T15:57:38.802143Z","shell.execute_reply":"2022-10-12T15:57:38.822174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collect_metadata(path):\n    files = glob.glob(path + '*.hdf5')\n    print(f'Found {len(files)} files in {path}.')\n    \n    data = []\n    for f in tqdm(files):\n        stats = os.stat(f)\n        \n        fid = os.path.basename(f).split('.')[0]\n    \n        with h5py.File(f, 'r') as file:\n            file = file[fid]\n            SFT_H = file['H1']['SFTs']\n            SFT_L = file['L1']['SFTs']\n            SFT_H_nunique = len(np.unique(SFT_H))\n            SFT_L_nunique = len(np.unique(SFT_L))\n            \n            \n        d = {\n            'id': fid,\n            'mode': stats.st_mode,\n            'ino': stats.st_ino,\n            'dev': stats.st_dev,\n            'nlink': stats.st_nlink,\n            'uid': stats.st_uid,\n            'gid': stats.st_gid,\n            'size': stats.st_size,\n            'atime': stats.st_atime,\n            'mtime': stats.st_mtime,\n            'ctime': stats.st_ctime,\n            'SFT_H_nunique': SFT_H_nunique,\n            'SFT_L_nunique': SFT_L_nunique,\n            'SFT_H_height': SFT_H.shape[0],\n            'SFT_H_width': SFT_H.shape[1],\n            'SFT_L_height': SFT_L.shape[0],\n            'SFT_L_width': SFT_L.shape[1],\n        }\n        data.append(d)\n        \n    return pd.DataFrame(data)","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:22:41.68969Z","iopub.execute_input":"2022-10-12T16:22:41.69022Z","iopub.status.idle":"2022-10-12T16:22:41.703363Z","shell.execute_reply.started":"2022-10-12T16:22:41.69018Z","shell.execute_reply":"2022-10-12T16:22:41.701467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = collect_metadata('../input/g2net-detecting-continuous-gravitational-waves/train/')","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:22:42.216655Z","iopub.execute_input":"2022-10-12T16:22:42.217206Z","iopub.status.idle":"2022-10-12T16:24:52.068947Z","shell.execute_reply.started":"2022-10-12T16:22:42.21715Z","shell.execute_reply":"2022-10-12T16:24:52.067462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = collect_metadata('../input/g2net-detecting-continuous-gravitational-waves/test/')","metadata":{"execution":{"iopub.status.busy":"2022-10-12T15:59:41.100448Z","iopub.execute_input":"2022-10-12T15:59:41.100957Z","iopub.status.idle":"2022-10-12T15:59:44.048636Z","shell.execute_reply.started":"2022-10-12T15:59:41.100915Z","shell.execute_reply":"2022-10-12T15:59:44.047458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = sorted(list(train['id'].values) + list(test['id'].values))\norder_df = pd.DataFrame()\norder_df['id'] = ids\norder_df = order_df.reset_index().rename({0: 'alph_order_i'}, axis=1)\n\ntrain = train.merge(order_df, on='id', how='left')\ntest = test.merge(order_df, on='id', how='left')\n\norder_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:15:50.738934Z","iopub.execute_input":"2022-10-12T16:15:50.739584Z","iopub.status.idle":"2022-10-12T16:15:50.795571Z","shell.execute_reply.started":"2022-10-12T16:15:50.739534Z","shell.execute_reply":"2022-10-12T16:15:50.794126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:15:53.466403Z","iopub.execute_input":"2022-10-12T16:15:53.466892Z","iopub.status.idle":"2022-10-12T16:15:53.475326Z","shell.execute_reply.started":"2022-10-12T16:15:53.466855Z","shell.execute_reply":"2022-10-12T16:15:53.473976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(train_labels, on='id', how='right')\ntrain = train[train['target'] != -1]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:16:14.985086Z","iopub.execute_input":"2022-10-12T16:16:14.985595Z","iopub.status.idle":"2022-10-12T16:16:15.002484Z","shell.execute_reply.started":"2022-10-12T16:16:14.985542Z","shell.execute_reply":"2022-10-12T16:16:15.001178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train_features.csv', index=False)\ntest.to_csv('test_features.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = [i for i in train.columns if i not in ['id', 'target']]\nkf = KFold(n_splits=10, shuffle=True, random_state=0)\npreds = []\nauc_buf = []\n\nfor fold_i, (train_idx, valid_idx) in enumerate(kf.split(train, train['target'])):\n    print(f'Fold {fold_i}')\n    \n    X_train = train.iloc[train_idx][feature_names]\n    y_train = train.iloc[train_idx]['target']\n    \n    X_valid = train.iloc[valid_idx][feature_names]\n    y_valid = train.iloc[valid_idx]['target']\n    \n    model = LGBMClassifier(n_estimators=5000)  \n    model.fit(\n        X_train,\n        y_train,\n        eval_set=[(X_valid, y_valid)],\n        eval_metric='auc',\n        callbacks=[\n            lgb.early_stopping(100),\n            lgb.log_evaluation(100)\n        ]\n    )\n    \n    p_valid = model.predict_proba(X_valid)[:, 1]\n    auc = roc_auc_score(y_valid, p_valid)\n    auc_buf.append(auc)\n    \n    p = model.predict_proba(test[feature_names])[:, 1]\n    preds.append(p)\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:17:21.036781Z","iopub.execute_input":"2022-10-12T16:17:21.037415Z","iopub.status.idle":"2022-10-12T16:17:22.281585Z","shell.execute_reply.started":"2022-10-12T16:17:21.037369Z","shell.execute_reply":"2022-10-12T16:17:22.280291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'AUC: {np.mean(auc_buf):.6f} +/- {np.std(auc_buf):.6f}')","metadata":{"execution":{"iopub.status.busy":"2022-10-12T16:17:28.06278Z","iopub.execute_input":"2022-10-12T16:17:28.06331Z","iopub.status.idle":"2022-10-12T16:17:28.070175Z","shell.execute_reply.started":"2022-10-12T16:17:28.063267Z","shell.execute_reply":"2022-10-12T16:17:28.069118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = pd.DataFrame()\nsubm['id'] = test['id'].values\nsubm['target'] = np.mean(preds, axis=0)\nsubm.to_csv('submission.csv', index=False)\nprint(subm.shape)\nsubm.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-10-12T15:57:54.990872Z","iopub.execute_input":"2022-10-12T15:57:54.991386Z","iopub.status.idle":"2022-10-12T15:57:55.03055Z","shell.execute_reply.started":"2022-10-12T15:57:54.991346Z","shell.execute_reply":"2022-10-12T15:57:55.029404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm['target'].mean(), subm['target'].min(), subm['target'].max(), subm['target'].std()","metadata":{"execution":{"iopub.status.busy":"2022-10-12T15:57:57.101692Z","iopub.execute_input":"2022-10-12T15:57:57.102438Z","iopub.status.idle":"2022-10-12T15:57:57.112641Z","shell.execute_reply.started":"2022-10-12T15:57:57.102391Z","shell.execute_reply":"2022-10-12T15:57:57.111418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}