{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net - Record Generation\n\nPrepare an initial set of TF records for a first baseline attempt.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"add on \nLAU_01B\nG2Net-[Signal]-Generate","metadata":{}},{"cell_type":"code","source":"import os\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport h5py\nimport tensorflow as tf\nprint(tf.__version__)\nfrom sklearn.model_selection import KFold\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2022-11-28T21:23:51.56372Z","iopub.execute_input":"2022-11-28T21:23:51.565031Z","iopub.status.idle":"2022-11-28T21:23:58.173498Z","shell.execute_reply.started":"2022-11-28T21:23:51.564895Z","shell.execute_reply":"2022-11-28T21:23:58.172306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor root, folders, filenames in os.walk('/kaggle/input'):\n   print(root, folders)","metadata":{"execution":{"iopub.status.busy":"2022-11-28T21:23:58.175304Z","iopub.execute_input":"2022-11-28T21:23:58.176205Z","iopub.status.idle":"2022-11-28T21:24:02.156171Z","shell.execute_reply.started":"2022-11-28T21:23:58.176174Z","shell.execute_reply":"2022-11-28T21:24:02.154683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport shutil\n\n# Providing the folder path\norigin = '/kaggle/input/g2net-detecting-continuous-gravitational-waves/train/'\n#origin=Path(origin)\ntarget = '../temp/train/'\ntemp='../temp/'\nif not os.path.exists(temp):\n    os.mkdir(temp)\nif not os.path.exists(target):\n    os.mkdir(target)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-28T21:24:02.157891Z","iopub.execute_input":"2022-11-28T21:24:02.158581Z","iopub.status.idle":"2022-11-28T21:24:02.164208Z","shell.execute_reply.started":"2022-11-28T21:24:02.158544Z","shell.execute_reply":"2022-11-28T21:24:02.163014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetching the list of all the files\nfiles = os.listdir(origin)\n\n# Fetching all the files to directory\nfor file_name in files:\n    shutil.copy(origin+file_name, target+file_name)\nprint(\"Files are copied successfully\")","metadata":{"execution":{"iopub.status.busy":"2022-11-28T21:24:02.166664Z","iopub.execute_input":"2022-11-28T21:24:02.167324Z","iopub.status.idle":"2022-11-28T21:26:53.266821Z","shell.execute_reply.started":"2022-11-28T21:24:02.167289Z","shell.execute_reply":"2022-11-28T21:26:53.264395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_temporar = os.listdir(target)\nfiles_temporar[555]","metadata":{"execution":{"iopub.status.busy":"2022-11-28T21:26:53.269106Z","iopub.execute_input":"2022-11-28T21:26:53.269595Z","iopub.status.idle":"2022-11-28T21:26:53.284351Z","shell.execute_reply.started":"2022-11-28T21:26:53.269534Z","shell.execute_reply":"2022-11-28T21:26:53.282686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_on = '/kaggle/input/g2netsignalgenerate/'\nfiles_add_on = os.listdir(add_on)\nfor file_name in files_add_on:\n    shutil.copy(add_on+file_name, target+file_name)","metadata":{"execution":{"iopub.status.busy":"2022-11-28T21:26:53.285413Z","iopub.execute_input":"2022-11-28T21:26:53.285691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"/kaggle/input/g2net-detecting-continuous-gravitational-waves/train_labels.csv\")\nlen(train)\ntrain.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\n#df_add=['target'] = data['activity'].apply(lambda x: 'skin diving' if 'skin diving' in x else x)\nlen(df_add)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_add['target'] = df_add['id'].apply(lambda x: 0 if 'noise' in str(x) else 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_add.tail(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_add.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.concat([train, df_add])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add_on = '/kaggle/input/pyfstat-noise-and-some-signals/noise/'\nadd_on = '/kaggle/input/g2netnonstationary-dataset/noise/'\nfiles_add_on = os.listdir(add_on)\nfor file_name in files_add_on:\n    shutil.copy(add_on+file_name, target+file_name)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\n#df_add=['target'] = data['activity'].apply(lambda x: 'skin diving' if 'skin diving' in x else x)\ndf_add['target']=0\ntrain=pd.concat([train, df_add])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add_on = '/kaggle/input/pyfstat-noise-and-some-signals/noise/'\nadd_on = '/kaggle/input/g2netgaussian-noise-dataset/noise/'\nfiles_add_on = os.listdir(add_on)\nfor file_name in files_add_on:\n    shutil.copy(add_on+file_name, target+file_name)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\n#df_add=['target'] = data['activity'].apply(lambda x: 'skin diving' if 'skin diving' in x else x)\ndf_add['target']=0\ntrain=pd.concat([train, df_add])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_on","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add_on = '/kaggle/input/pyfstat-noise-and-some-signals/signal/'\nadd_on = '/kaggle/input/g2netnonstationary-dataset/signal/'\nfiles_add_on = os.listdir(add_on)\nfor file_name in files_add_on:\n    shutil.copy(add_on+file_name, target+file_name)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\n#df_add=['target'] = data['activity'].apply(lambda x: 'skin diving' if 'skin diving' in x else x)\ndf_add['target']=1\ntrain=pd.concat([train, df_add])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add_on = '/kaggle/input/pyfstat-noise-and-some-signals/signal/'\nadd_on = '/kaggle/input/g2netgaussian-noise-dataset/signal/'\nfiles_add_on = os.listdir(add_on)\nfor file_name in files_add_on:\n    shutil.copy(add_on+file_name, target+file_name)\ndf_add=pd.DataFrame()   \ndf_add=pd.DataFrame(files_add_on)\ndf_add=df_add.rename(columns={0: \"id\"})\n#df_add=['target'] = data['activity'].apply(lambda x: 'skin diving' if 'skin diving' in x else x)\ndf_add['target']=1\ntrain=pd.concat([train, df_add])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['id']=train['id'].str.replace('.hdf5', '')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    train = train.drop(train[train.id.str.contains('.csv')].index)\nexcept:\n    print('do nothing')\n#df = df.drop(df[df.score < 50].index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['id'].str.contains('.csv')]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ls ../temp/train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['calea']='../temp/train/'+train.id+'.hdf5'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utility to read hdf5 file\ndef read_data(file: Path):\n    with h5py.File(file, \"r\") as f:\n        file=Path(file)\n        filename = file.stem\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        #freq_hz = list(f[\"frequency_Hz\"])\n        ###\n        h1_stft = h1[\"SFTs\"][()]\n        ###\n        #h1_timestamp = h1[\"timestamps_GPS\"][()]\n        h1_timestamp =1\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        #l1_timestamp = l1[\"timestamps_GPS\"][()]\n        l1_timestamp =2\n        return {\n            \"H1\": [h1_stft, h1_timestamp],\n            \"L1\": [l1_stft, l1_timestamp],\n            #\"freq_hz\": freq_hz\n        }\ndef read_data_2(file: Path):\n    with h5py.File(file, \"r\") as f:\n        file=Path(file)\n        filename = file.stem\n        filename = filename\n        f = f[filename]\n        h1 = f[\"H1\"]\n        l1 = f[\"L1\"]\n        #freq_hz = list(f[\"frequency_Hz\"])\n        ###\n        h1_stft = h1[\"SFTs\"][()]\n        ###\n        #h1_timestamp = h1[\"timestamps_GPS\"][()]\n        h1_timestamp =1\n        # H2 data\n        l1_stft = l1[\"SFTs\"][()]\n        #l1_timestamp = l1[\"timestamps_GPS\"][()]\n        l1_timestamp =2\n        return h1_stft,l1_stft\n         \n\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filter=[]\nfor filename in train.calea:\n    H1_SFTs, L1_SFTs = read_data_2(filename)\n#     #print('-'*25)\n#     print(H1_SFTs.shape)\n#     print(L1_SFTs.shape)\n#     print(filename)\n    filter.append({\n        'path': filename,\n        'size1h': (H1_SFTs.shape)[1],\n        'size2h': (H1_SFTs.shape)[0],\n        'size1l': (L1_SFTs.shape)[1],\n        'size2l': (L1_SFTs.shape)[0],\n    })\n    #print('H1_SFTs     ', H1_SFTs.shape, '   H1_timestamp', H1_timestamp.shape)\n    #print('Frequencies ', 'max:',frequencies.max().round(3),  '   min:',frequencies.min().round(3))\n    print()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filter_df=pd.DataFrame(filter)\nfilter_df.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train_labels.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"code","source":"class Config:\n    # setting\n    seed = 887    \n    n_jobs = 4\n    # input data\n    base_dir = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\"\n#     path_train = base_dir + \"train_labels.csv\"\n#     train_dir = base_dir + \"train/\"\n    \n    path_train =\"/kaggle/working/train_labels.csv\"\n    train_dir = target\n    \n    path_test = base_dir + \"sample_submission.csv\"\n    test_dir = base_dir + \"test/\"\n    # output\n    output_dir = \"/kaggle/working/\"\n    output_dir_train = output_dir + \"train/\"\n    output_dir_test = output_dir + \"test/\"\n    name_train = \"g2net-train-orig\"\n    name_test = \"g2net-test\"\n    shard_train = 12\n    shard_test = 80\n    # spectrum\n    ts_max = 4320\n    img_size = 360, 360\n    reshape_size = (*img_size, ts_max // img_size[1])\n    \n\ncfg = Config()\nos.makedirs(cfg.output_dir, exist_ok=True)\nos.makedirs(cfg.output_dir_train, exist_ok=True)\nos.makedirs(cfg.output_dir_test, exist_ok=True)\ncfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts_max = 4320\nimg_size = 360, 360\nreshape_size = (*img_size, ts_max // img_size[1])\nprint(reshape_size)\nprint(*img_size,ts_max // img_size[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper function","metadata":{"execution":{"iopub.status.busy":"2022-11-07T21:10:41.714639Z","iopub.execute_input":"2022-11-07T21:10:41.715065Z","iopub.status.idle":"2022-11-07T21:10:41.721334Z","shell.execute_reply.started":"2022-11-07T21:10:41.715029Z","shell.execute_reply":"2022-11-07T21:10:41.720524Z"}}},{"cell_type":"code","source":"def add_shard(data, n_shards, shuffle=True):\n    \"\"\"Add shard number\"\"\"\n    kf = KFold(n_splits=n_shards, shuffle=shuffle, random_state=cfg.seed if shuffle else None)\n    data[\"shard\"] = -1\n    for i, (_, iloc_test) in enumerate(kf.split(data)):\n        ind_test = data.iloc[iloc_test].index\n        data.loc[ind_test, \"shard\"] = i","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_arr(arr):\n    try:\n        num_freq, num_ts = arr.shape\n        t = np.zeros(shape=(num_freq, cfg.ts_max), dtype=arr.dtype)\n        num_ts = min(num_ts, cfg.ts_max)\n        t[:, :num_ts] = arr[:, :num_ts]\n        # power\n        t = t.real ** 2 + t.imag ** 2\n        # normalize by mean over all entries\n        t = t / np.mean(t)\n        # group neighboring timestamps in 3rd dimension\n        t = t.reshape((cfg.reshape_size))\n        # take average over 3rd dimension (neighboring ts)\n        img = np.mean(t, axis=2)\n\n    except:\n        img = np.empty((cfg.img_size), dtype=np.float32)\n        print(\"eroare\")    \n    return img\n\n\ndef load_spectra(path_file):\n    with h5py.File(path_file, 'r') as f:\n        rec_id = list(f.keys())[0]\n        h1 = np.array(f[f\"{rec_id}/H1/SFTs\"]) * 1e22\n        h1 =h1[0:360,:]\n        h1 = process_arr(h1)\n        l1 = np.array(f[f\"{rec_id}/L1/SFTs\"]) * 1e22\n        l1 =l1[0:360,:]\n        l1 = process_arr(l1)\n    return np.asarray([h1, l1]).astype('float32')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\n# def _float_feature(value):\n#     \"\"\"Returns a float_list from a float / double.\"\"\"\n#     return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\n\ndef to_records(img_bytes, id_bytes, target_int):\n    feature = {\n        'img': _bytes_feature(img_bytes),\n        'id': _bytes_feature(id_bytes),\n    }\n    if target_int is not None:\n        feature['target'] = _int64_feature(target_int)\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef create_record(row, input_dir):\n    path_file = input_dir + f\"{row.id}.hdf5\"\n    img = load_spectra(path_file)\n    rec = to_records(\n        img_bytes=tf.io.serialize_tensor(img).numpy(),\n        id_bytes=row.id.encode('utf-8'),\n        target_int=getattr(row, \"target\", None),\n    )\n    return rec","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_shard(df, shard, input_dir, output_dir, name):\n    fname = f\"{name}_shard-{shard}_size-{len(df)}.tfrec\"\n    path_output = os.path.join(output_dir, fname)\n    print(f\"shard {shard}: {path_output}\")\n    cnt = 0\n    with tf.io.TFRecordWriter(path_output) as f:\n        for row in df.itertuples():\n            print(row)\n            rec = create_record(row, input_dir)\n            f.write(rec.SerializeToString())\n            cnt += 1\n    return cnt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create record","metadata":{}},{"cell_type":"markdown","source":"## Training dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(cfg.path_train)\ntrain.head(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[train[\"target\"] >= 0]\nadd_shard(train, cfg.shard_train)\ntrain = train.sample(frac=1)\ntrain.id=train.id.str.replace('.hdf5','')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sort_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_args = (cfg.train_dir, cfg.output_dir_train, cfg.name_train)\nresults = Parallel(n_jobs=cfg.n_jobs, verbose=1, backend='threading')(\n    delayed(write_shard)(df, shard, *train_args) for shard, df in train.groupby(\"shard\")\n)\ntrain.sort_index().to_csv(cfg.output_dir + \"/summary_train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test dataset","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(cfg.path_test)\ndel test[\"target\"]\nadd_shard(test, cfg.shard_test, shuffle=False)\ntest","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_args = (cfg.test_dir, cfg.output_dir_test, cfg.name_test)\nresults = Parallel(n_jobs=cfg.n_jobs, verbose=1, backend='threading')(\n    delayed(write_shard)(df, shard, *test_args) for shard, df in test.groupby(\"shard\")\n)\ntest.sort_index().to_csv(cfg.output_dir + \"/summary_test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"code","source":"!tree $cfg.output_dir","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/2104080/how-do-i-check-file-size-in-python\n\ndef convert_bytes(num):\n    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:\n        if num < 1024.0:\n            return \"%3.1f %s\" % (num, x)\n        num /= 1024.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for d in [cfg.output_dir_train, cfg.output_dir_test]:\n    bs = sum(os.stat(f).st_size for f in pathlib.Path(d).glob(\"*.*\"))\n    print(d, convert_bytes(bs))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}