{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport pandas as pd\nimport numpy as np\nimport h5py\nimport glob\nimport tensorflow as tf\nimport skimage.transform as st\n#import gc\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:44:47.314372Z","iopub.execute_input":"2022-12-31T22:44:47.314811Z","iopub.status.idle":"2022-12-31T22:44:54.601033Z","shell.execute_reply.started":"2022-12-31T22:44:47.314725Z","shell.execute_reply":"2022-12-31T22:44:54.599709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def avgFreq(y):\n    '''Averages frequency over all data points\n    '''\n    x = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\"\n    z = x+\"train/\"+\"{}.hdf5\".format(y)\n    A=np.array(h5py.File(z,\"r\")[y][\"frequency_Hz\"])\n    return np.mean(A)\n\ndef makeImg(y):\n    '''makes a compressed image containing both H and L data\n    '''\n    x = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\"\n    z = \"\".join([x, \"train/{}.hdf5\".format(y)])\n    img = np.empty((2, 360, 128), dtype=np.float16)\n    with h5py.File(z, 'r') as f:\n        #g = f[y]\n\n        for ch, s in enumerate(['H1', 'L1']):\n            #a = f[y][s]['SFTs'][:, :4096] * 1e22  # Fourier coefficient complex64\n\n            #p = a.real**2 + a.imag**2  # power\n            p = np.abs(f[y][s]['SFTs'][:, :4096] * 1e22)**2\n            p /= np.mean(p)  # normalize\n            p = np.mean(p.reshape(360, 128, 32), axis=2)  # compress 4096 -> 128\n\n            img[ch] = p\n\n    return img\n\ndef display(z):\n    '''makes a compressed image containing both H and L data\n    '''\n    y = z.split('/')[-1][:-5]\n    img = np.empty((2, 360, 128), dtype=np.float16)\n    with h5py.File(z, 'r') as f:\n        #g = f[y]\n\n        for ch, s in enumerate(['H1', 'L1']):\n            #a = f[y][s]['SFTs'][:, :4096] * 1e22  # Fourier coefficient complex64\n\n            #p = a.real**2 + a.imag**2  # power\n            p = np.abs(f[y][s]['SFTs'][:, :4096] * 1e22)**2\n            p /= np.mean(p)  # normalize\n            p = np.mean(p.reshape(360, 128, 32), axis=2)  # compress 4096 -> 128\n\n            img[ch] = p\n\n    return img\n\ndef plotImg(img):\n    '''plots image\n    '''\n    plt.figure(figsize=(13, 8))\n    plt.xlabel('time')\n    plt.ylabel('frequency')\n    plt.imshow(img[0, :]+img[1,:])\n    plt.colorbar()\n    plt.show()\n    \ndef comp(img):\n    '''compresses image so it fits in the model\n    '''\n    return st.resize(img,(2,128,128))\n\ndef build_dataset():\n    '''create tensorflow data set (the fun part)'''\n    x = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/\"\n    #read csv of training data\n    trainingLabels = pd.read_csv(\"\".join((x,'train_labels.csv')))\n    trainingLabels = trainingLabels[trainingLabels['target'].isin([0,1])]\n    #create mapping to translate names to targets\n    name_dict = dict(zip(trainingLabels['id'],trainingLabels['target']))\n    #under sample the positive samples\n    myList = np.concatenate((\n        np.random.choice(trainingLabels[trainingLabels['target']==1]['id'], 300),\n        trainingLabels[trainingLabels['target']==0]['id'].to_numpy()),\n        axis=0)\n    #strip duplicates in the positive set\n    myList = np.unique(myList)\n    # create images and targets from names, then pack into a dataset\n    my_ds = tf.data.Dataset.from_tensor_slices((np.array([comp(makeImg(name)) for name in myList]), \n                                                np.array([name_dict[name] for name in myList], dtype=np.float16)))\n    my_ds = my_ds.shuffle(len(myList))\n    return my_ds\n\ndef predictOnName(name):\n    \"\"\"make prediction on a given name in the test dataset\"\"\"\n    #file path the test images\n    url = '/kaggle/input/g2net-detecting-continuous-gravitational-waves/test/{}.hdf5'\n    #make predictions, returning the one that corresponds to presence of GW\n    a = tf.nn.softmax(model.predict(np.array([comp(display(url.format(name)))]))).numpy()\n    a = a[:,1][0]\n    yield a","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:44:54.603101Z","iopub.execute_input":"2022-12-31T22:44:54.603705Z","iopub.status.idle":"2022-12-31T22:44:54.625764Z","shell.execute_reply.started":"2022-12-31T22:44:54.60367Z","shell.execute_reply":"2022-12-31T22:44:54.624496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data augmentation layer\ndata_augmentation = tf.keras.Sequential([\n  tf.keras.layers.RandomFlip(\"horizontal_and_vertical\"),\n  tf.keras.layers.RandomZoom(0.015)\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:44:54.6273Z","iopub.execute_input":"2022-12-31T22:44:54.627652Z","iopub.status.idle":"2022-12-31T22:44:56.210063Z","shell.execute_reply.started":"2022-12-31T22:44:54.627614Z","shell.execute_reply":"2022-12-31T22:44:56.208438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_ds = build_dataset()","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:44:56.213208Z","iopub.execute_input":"2022-12-31T22:44:56.214036Z","iopub.status.idle":"2022-12-31T22:47:16.787971Z","shell.execute_reply.started":"2022-12-31T22:44:56.21399Z","shell.execute_reply":"2022-12-31T22:47:16.784671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# augment the data\nmy_ds = my_ds.map(lambda x, y: (data_augmentation(x), y),  num_parallel_calls=tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:47:16.792521Z","iopub.execute_input":"2022-12-31T22:47:16.793248Z","iopub.status.idle":"2022-12-31T22:47:17.164793Z","shell.execute_reply.started":"2022-12-31T22:47:16.793184Z","shell.execute_reply":"2022-12-31T22:47:17.162921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#build a simple model\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Conv2D(16, 3, padding='same', activation='relu'),\n    tf.keras.layers.MaxPooling2D((2,2)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(2)\n])\n\n#compile the model\nmodel.compile(optimizer='adam',\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:47:17.16651Z","iopub.execute_input":"2022-12-31T22:47:17.166829Z","iopub.status.idle":"2022-12-31T22:47:17.216435Z","shell.execute_reply.started":"2022-12-31T22:47:17.1668Z","shell.execute_reply":"2022-12-31T22:47:17.214464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch datasets\nmy_ds = my_ds.batch(32)\n#fit the model\noutput = model.fit(my_ds,epochs=30)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:47:17.218244Z","iopub.execute_input":"2022-12-31T22:47:17.218677Z","iopub.status.idle":"2022-12-31T22:47:33.770732Z","shell.execute_reply.started":"2022-12-31T22:47:17.218643Z","shell.execute_reply":"2022-12-31T22:47:33.769535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#URL for the test data CSV\nsampleSub = '/kaggle/input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv'\n#extract the id's\nf=pd.read_csv(sampleSub)['id'].to_numpy()\n#make a generator\nsomeGenerator = (next(predictOnName(x)) for x in f)\n#apply the generator\nmy_array = np.empty(len(f))\nfor i, el in enumerate(someGenerator): \n    my_array[i] = el","metadata":{"execution":{"iopub.status.busy":"2022-12-31T22:47:33.772945Z","iopub.execute_input":"2022-12-31T22:47:33.773353Z","iopub.status.idle":"2022-12-31T23:42:46.318771Z","shell.execute_reply.started":"2022-12-31T22:47:33.773316Z","shell.execute_reply":"2022-12-31T23:42:46.313838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#set target values\nsampleSub= pd.DataFrame({'id':f, 'target':my_array})\nsampleSub.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-31T23:42:46.325442Z","iopub.execute_input":"2022-12-31T23:42:46.326047Z","iopub.status.idle":"2022-12-31T23:42:46.429542Z","shell.execute_reply.started":"2022-12-31T23:42:46.326004Z","shell.execute_reply":"2022-12-31T23:42:46.427131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#write results to disk\nsampleSub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T23:42:46.431658Z","iopub.execute_input":"2022-12-31T23:42:46.432036Z","iopub.status.idle":"2022-12-31T23:42:46.478243Z","shell.execute_reply.started":"2022-12-31T23:42:46.432002Z","shell.execute_reply":"2022-12-31T23:42:46.476596Z"},"trusted":true},"execution_count":null,"outputs":[]}]}