{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Instalación de librerías ","metadata":{}},{"cell_type":"code","source":"!pip install riroriro\n!pip install visualkeras\n!pip install git+https://github.com/PyFstat/PyFstat@python37\n#!pip install tensorflow-addons==0.16.1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Importar de librerías  ","metadata":{}},{"cell_type":"code","source":"import os\nimport h5py\nimport gc\nimport glob\nimport math\nimport random\nimport warnings\nimport pyfstat\nimport librosa\nimport librosa.display\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport visualkeras\nimport riroriro.inspiralfuns as ins\nimport riroriro.mergerfirstfuns as me1\nimport riroriro.matchingfuns as mat\nimport riroriro.mergersecondfuns as me2\nimport tensorflow as tf\nfrom scipy.signal import istft\n\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\n\nfrom pathlib import Path\nfrom scipy import stats\nfrom tqdm.notebook import tqdm\nfrom scipy import signal\nimport matplotlib.pyplot as plt\nfrom IPython.display import HTML, display\nfrom tensorflow.keras.utils import plot_model\nfrom sklearn.model_selection   import train_test_split\n\nsns.set_theme()\n%matplotlib inline \nwarnings.filterwarnings('ignore')\ndisplay(HTML('<style>.font-family:verdana; word-spacing:1.5px;</style>'))","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-07T21:15:13.166298Z","iopub.execute_input":"2023-01-07T21:15:13.166715Z","iopub.status.idle":"2023-01-07T21:15:13.185424Z","shell.execute_reply.started":"2023-01-07T21:15:13.166673Z","shell.execute_reply":"2023-01-07T21:15:13.184156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Importar datos\n\nEl dataset esta compuesto por datos de frecuencia de tiempo de dos interferómetros de ondas gravitacionales (LIGO Hanford y LIGO Livingston). Cada muestra de datos contiene ruido real o simulado y posiblemente una señal de onda gravitacional (CW) continua simulada.\n\nCada muestra se compone de un conjunto de transformadas de Fourier de tiempo corto (SFT) y las marcas de tiempo de GPS correspondientes para cada interferómetro. Las SFT no siempre son contiguas en el tiempo, ya que los interferómetros no están continuamente en línea.\n\nLas señales simuladas están presentes durante toda la duración del conjunto de SFT en ambos detectores. Las señales se caracterizan por la ubicación y orientación de la fuente astrofísica hipotética, así como por dos parámetros intrínsecos: frecuencia y spin-down. En total hay ocho parámetros que han sido aleatorizados. Estos no se proporcionan como parte de los datos. Las amplitudes típicas de las señales resultantes son uno o dos órdenes de magnitud inferiores a la amplitud del ruido del detector.\n\n### Archivos\n\n[train/|test/] - Carpetas que contienen los datos de entrenamiento y test.\n\ntrain_labels.csv: Un archivo que contiene las target, etiquetas; 1 si los datos contienen una onda gravitacional, 0 caso contrario y una pequeña cantidad de archivos con la etiqueta -1. Actualmente, los físicos no pueden determinar el estado de estos archivos.\n\nsample_submission.csv: Un archivo de envío de muestra en el formato correcto; su tarea es, para cada archivo en la test/carpeta, predecir la probabilidad [0, 1]de que contenga una señal de onda gravitacional continua.","metadata":{}},{"cell_type":"code","source":"DATA_PATH = Path('../input/g2net-detecting-continuous-gravitational-waves')\nTRAIN_PATH = DATA_PATH/'train'\nTEST_PATH = DATA_PATH/'test'","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:20.742955Z","iopub.execute_input":"2023-01-07T21:15:20.743723Z","iopub.status.idle":"2023-01-07T21:15:20.748563Z","shell.execute_reply.started":"2023-01-07T21:15:20.743686Z","shell.execute_reply":"2023-01-07T21:15:20.747162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_df = pd.read_csv(DATA_PATH/'train_labels.csv')\nlabels_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:23.143537Z","iopub.execute_input":"2023-01-07T21:15:23.144492Z","iopub.status.idle":"2023-01-07T21:15:23.168207Z","shell.execute_reply.started":"2023-01-07T21:15:23.144454Z","shell.execute_reply":"2023-01-07T21:15:23.166854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Explorando la estructura de los conjuntos de datos","metadata":{}},{"cell_type":"code","source":"def allkeys(obj):\n    \n    \"\"\"Encuentre recursivamente todas las claves en un h5py.Group.\"\"\"\n    keys = (obj.name,)\n    if isinstance(obj, h5py.Group):\n        for key, value in obj.items():\n            if isinstance(value, h5py.Group):\n                keys = keys + allkeys(value)\n            else:\n                keys = keys + (value.name,)\n    return keys","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:27.211326Z","iopub.execute_input":"2023-01-07T21:15:27.211747Z","iopub.status.idle":"2023-01-07T21:15:27.219224Z","shell.execute_reply.started":"2023-01-07T21:15:27.211713Z","shell.execute_reply":"2023-01-07T21:15:27.217757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Impresión dos muestras de datos una que contiene una onda gravitacional (1) y otra solo ruido (0).","metadata":{}},{"cell_type":"code","source":"print(labels_df[labels_df['id']=='001121a05'])\nprint(labels_df[labels_df['id']=='01bcf6533'])","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:30.014477Z","iopub.execute_input":"2023-01-07T21:15:30.014955Z","iopub.status.idle":"2023-01-07T21:15:30.027417Z","shell.execute_reply.started":"2023-01-07T21:15:30.014919Z","shell.execute_reply":"2023-01-07T21:15:30.026331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_example_with_signal_path = TRAIN_PATH/'cc561e4fc.hdf5'# target = 1\ntrain_example_without_signal_path = TRAIN_PATH/'fb6db0d08.hdf5' # target = 0","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:32.461938Z","iopub.execute_input":"2023-01-07T21:15:32.462511Z","iopub.status.idle":"2023-01-07T21:15:32.466515Z","shell.execute_reply.started":"2023-01-07T21:15:32.462478Z","shell.execute_reply":"2023-01-07T21:15:32.465505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hf = h5py.File(train_example_with_signal_path, 'r')\nprint(allkeys(hf)) # print of architecture of dataset.","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:34.944999Z","iopub.execute_input":"2023-01-07T21:15:34.945736Z","iopub.status.idle":"2023-01-07T21:15:34.956319Z","shell.execute_reply.started":"2023-01-07T21:15:34.945695Z","shell.execute_reply":"2023-01-07T21:15:34.955368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hf.keys()","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:37.39184Z","iopub.execute_input":"2023-01-07T21:15:37.393379Z","iopub.status.idle":"2023-01-07T21:15:37.401707Z","shell.execute_reply.started":"2023-01-07T21:15:37.39331Z","shell.execute_reply":"2023-01-07T21:15:37.400553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Estructura de train data y test\n\n\n- <span style=\"font-family:verdana; word-spacing:1.5px;\"> `ID` Es el grupo superior del archivo HDF5 y vincula el punto de datos a su etiqueta en el csv de `train_labels` (grupo).\n\n- <span style=\"font-family:verdana; word-spacing:1.5px;\"> `frecuencia_Hz` Contiene las frecuencias de rango medidas por los dectores (conjunto de datos).\n\n- <span style=\"font-family:verdana; word-spacing:1.5px;\"> `H1` Contiene los datos para el decector LIGO Hanford (grupo).\n\n    - <span style=\"font-family:verdana; word-spacing:1.5px;\"> `SFTs` Son las amplitudes de las transformadas de Fourier de tiempo corto para cada marca de tiempo en cada frecuencia (conjunto de datos).\n    - <span style=\"font-family:verdana; word-spacing:1.5px;\"> `timestamps` Contiene las marcas de tiempo para la medición (conjunto de datos).\n    \n- <span style=\"font-family:verdana; word-spacing:1.5px;\">`L1` Contiene los datos para el decector LIGO Livingston (grupo).\n\n    - <span style=\"font-family:verdana; word-spacing:1.5px;\"> `SFTs` Son las amplitudes de las transformadas de Fourier de tiempo corto para cada marca de tiempo en cada frecuencia (conjunto de datos).    \n    - <span style=\"font-family:verdana; word-spacing:1.5px;\"> `timestamps` Contiene las marcas de tiempo para la medición (conjunto de datos).\n    \n","metadata":{}},{"cell_type":"markdown","source":"![](https://i.imgur.com/M6xfOri.png)","metadata":{}},{"cell_type":"markdown","source":"### Distribución de train y test","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,6))\nsns.barplot(['Train', 'Test'], [len(os.listdir(TRAIN_PATH)), len(os.listdir(TEST_PATH))]);\nplt.title(f'Train & test', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xlabel('Categoría', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:26:29.418532Z","iopub.execute_input":"2023-01-07T21:26:29.418931Z","iopub.status.idle":"2023-01-07T21:26:29.575016Z","shell.execute_reply.started":"2023-01-07T21:26:29.418899Z","shell.execute_reply":"2023-01-07T21:26:29.573722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Porcentaje de train y test\n","metadata":{}},{"cell_type":"code","source":"test_set_train_set = len(os.listdir(TEST_PATH))/(len(os.listdir(TEST_PATH))+len(os.listdir(TRAIN_PATH)))\nprint(f\"test_set/dataset: {round(test_set_train_set*100,1)} %\")","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:29:30.27893Z","iopub.execute_input":"2023-01-07T21:29:30.279377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"La mayoría de las veces nos encontramos con divisiones de datos del 80 % para entrenamiento y 20 % para test, pero en nuestro caso es del 7% y el 93%, lo que indica generar datos propios.","metadata":{}},{"cell_type":"markdown","source":"### Distribución de etiquetas en train.","metadata":{}},{"cell_type":"code","source":"label_count  = labels_df['target'].value_counts()\nplt.figure(figsize=(10,8))\nsns.barplot(label_count.index, label_count.values, alpha=0.7)\nplt.title(f'Frecuencia de etiquetas en train', fontsize=12)\nplt.ylabel('Count', fontsize=12)\nplt.xlabel('label', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:29:18.395159Z","iopub.execute_input":"2023-01-07T21:29:18.396385Z","iopub.status.idle":"2023-01-07T21:29:18.535691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Etiquetas; \"1\" si los datos contienen la presencia de una onda gravitacional, \"0\" en caso contrario y una pequeña cantidad de \"-1\" que actualmente, los físicos no pueden determinar el estado de estos archivos por lo tanto los eliminaremos.","metadata":{}},{"cell_type":"markdown","source":"### Análisis de espectrograma\n\nUn espectrograma es una representación visual del espectro de frecuencias de una señal a medida que varía con el tiempo. Un espectrograma suele representarse como un mapa de calor, es decir, como una imagen cuya intensidad se muestra variando el color o el brillo.","metadata":{}},{"cell_type":"code","source":"def extract_data_from_hdf5(path, labels):\n\n    # Extrae datos del archivo hdf5 y los coloca en un dictado. También agrega la etiqueta.\n    \n    data = {}\n    \n    with h5py.File(path, \"r\") as f:\n\n        ID_key = list(f.keys())[0]\n\n        # Recuperar los datos de frecuencia\n        data['freq'] = np.array(f[ID_key]['frequency_Hz'])\n\n        # Recuperar los datos del detector de Livingston\n        data['L1_SFTs_amplitudes'] = np.array(f[ID_key]['L1']['SFTs'])\n        data['L1_ts'] = np.array(f[ID_key]['L1']['timestamps_GPS'])\n\n        # Recuperar los datos del detector Hanford\n        data['H1_SFTs_amplitudes'] = np.array(f[ID_key]['H1']['SFTs'])\n        data['H1_ts'] = np.array(f[ID_key]['H1']['timestamps_GPS'])\n        \n        # Obtenga la etiqueta de las etiquetas de entrenamiento si está en el conjunto de entrenamiento\n        data['label'] = labels.loc[labels.id==ID_key].target.item()\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:15:51.433296Z","iopub.execute_input":"2023-01-07T21:15:51.433694Z","iopub.status.idle":"2023-01-07T21:15:51.442408Z","shell.execute_reply.started":"2023-01-07T21:15:51.433661Z","shell.execute_reply":"2023-01-07T21:15:51.441478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_data_from_hdf5_reduced(path, labels):\n\n    # Extrae datos del archivo hdf5 y los coloca en un dictado. También agrega la etiqueta\n    \n    data = {}\n    \n    with h5py.File(path, \"r\") as f:\n\n        ID_key = list(f.keys())[0]\n\n        # Recuperar los datos de frecuencia\n        data['id'] = ID_key\n        data['freq'] = np.array(f[ID_key]['frequency_Hz'])\n\n        # Recuperar los datos del detector de Livingston\n        data['L1_ts'] = np.array(f[ID_key]['L1']['timestamps_GPS'])\n\n        # Recuperar los datos del detector Hanford\n        data['H1_ts'] = np.array(f[ID_key]['H1']['timestamps_GPS'])\n        \n        # Obtenga la etiqueta de las etiquetas de entrenamiento si está en el conjunto de entrenamiento\n        data['label'] = labels.loc[labels.id==ID_key].target.item()\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:16:05.110266Z","iopub.execute_input":"2023-01-07T21:16:05.111143Z","iopub.status.idle":"2023-01-07T21:16:05.119186Z","shell.execute_reply.started":"2023-01-07T21:16:05.111103Z","shell.execute_reply":"2023-01-07T21:16:05.117522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_spectograms(data):\n\n    # Muestra las amplitudes reales e imaginarias de los SFT como espectrogramas para ambos detectores\n    \n    fig, ax = plt.subplots(2, 2, figsize=(16, 10))\n    fig.suptitle(f\"Label {data['label']}\")\n\n    for ind, detector in enumerate(['L1', 'H1']):\n        ax[ind][0].set(xlabel=\"Timestamps [GPS]\",\n                         ylabel=\"Frequency [Hz]\",\n                         title=f\"{detector} - Real part\")\n        ax[ind][1].set(xlabel=\"Timestamps [GPS]\",\n                         ylabel=\"Frequency [Hz]\",\n                         title=f\"{detector} - Imaginary part\")\n        \n        \n        c0 = ax[ind][0].pcolormesh(data[f\"{detector}_ts\"], data['freq'],\n                                     data[f\"{detector}_SFTs_amplitudes\"].real)\n        c1 = ax[ind][1].pcolormesh(data[f\"{detector}_ts\"], data['freq'],\n                                     data[f\"{detector}_SFTs_amplitudes\"].imag)\n    \n        fig.colorbar(c0, ax=ax[ind][0])\n        fig.colorbar(c1, ax=ax[ind][1])\n        \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:37:14.094428Z","iopub.execute_input":"2023-01-07T21:37:14.094863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Trazado de espectrogramas\n\nGrafiquemos algunos espectrogramas. Uno con señal simulada y otro sin ella.","metadata":{}},{"cell_type":"code","source":"data = extract_data_from_hdf5(train_example_with_signal_path, labels_df)\nplot_spectograms(data)","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:16:12.955977Z","iopub.execute_input":"2023-01-07T21:16:12.956358Z","iopub.status.idle":"2023-01-07T21:16:16.494311Z","shell.execute_reply.started":"2023-01-07T21:16:12.956328Z","shell.execute_reply":"2023-01-07T21:16:16.492814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = extract_data_from_hdf5(train_example_without_signal_path, labels_df)\nplot_spectograms(data)","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:17:10.149617Z","iopub.execute_input":"2023-01-07T21:17:10.150029Z","iopub.status.idle":"2023-01-07T21:17:14.024886Z","shell.execute_reply.started":"2023-01-07T21:17:10.149996Z","shell.execute_reply":"2023-01-07T21:17:14.023132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Es muy difícil notar alguna diferencia.\n\n## Significado de las partes real e imaginaria de una Transformada de Fourier de Tiempo Corto.\n\n### Análisis de marca de tiempo\n\nDado que se simula la onda gravitacional continua, no estamos seguros de cómo se determina la longitud de la misma; consulte la generación de señales para obtener más detalles. Como resultado, asumimos que los datos de la marca de tiempo son relativamente insignificantes.\n\n### Extraer datos de marca de tiempo de los datos de entrenamiento","metadata":{}},{"cell_type":"code","source":"H1_timestamps, L1_timestamps, start_diff, labels, freq = ([] for i in range(5))\n\nfor p in tqdm(os.listdir(TRAIN_PATH), total=len(os.listdir(TRAIN_PATH))):\n    id_ = p.split('.')[0]\n    labels.append(labels_df.loc[labels_df.id==id_].target.item())\n    data = extract_data_from_hdf5(DATA_PATH/'train'/p, labels_df)\n    L1_timestamps.append(data['L1_ts'])\n    H1_timestamps.append(data['H1_ts'])\n    start_diff.append(data['L1_ts'][0] - data['H1_ts'][0])\n    freq.append(data['freq'])","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:48:27.774749Z","iopub.execute_input":"2023-01-07T21:48:27.775131Z","iopub.status.idle":"2023-01-07T21:50:50.505212Z","shell.execute_reply.started":"2023-01-07T21:48:27.775104Z","shell.execute_reply":"2023-01-07T21:50:50.504227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creando un marco de datos con etiquetas, longitud de L timestmps, longitud de H timestmps","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame({'label':labels, 'L1_timestamp_length':[len(i) for i in L1_timestamps], 'H1_timestamp_length':[len(i) for i in H1_timestamps], 'Differnce in start time between detectors':start_diff})\n# soltar los valores de las etiquetas = -1\ndf = df[df.label!=-1] ","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:48:12.493176Z","iopub.execute_input":"2023-01-07T21:48:12.494615Z","iopub.status.idle":"2023-01-07T21:48:12.531394Z","shell.execute_reply.started":"2023-01-07T21:48:12.494554Z","shell.execute_reply":"2023-01-07T21:48:12.529706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme()\n\nfig, ax = plt.subplots(1,3, figsize=(24,8))\nfig.suptitle(f\"In the plots the distribution of timestamps for both classes are shown; 1 indicates a simulated CW present and 0 not present\", fontsize=16)\nsns.histplot(\n        df, x=\"L1_timestamp_length\", hue=\"label\",\n        stat=\"density\", common_norm=False, bins=20, ax=ax[0], kde=True).set_title('Length of measurement for Livingston detector', fontsize=16);\n\nsns.histplot(\n        df, x=\"H1_timestamp_length\", hue=\"label\",\n        stat=\"density\", common_norm=False, bins=20, ax=ax[1], kde=True).set_title('Length of measurement for Hanford detector', fontsize=16);\n\nsns.histplot(\n        df, x=\"Differnce in start time between detectors\", hue=\"label\",\n        stat=\"density\", common_norm=False, bins=20, ax=ax[2], kde=True).set_title('Difference in starting timestamp between detectors', fontsize=16);","metadata":{"execution":{"iopub.status.busy":"2023-01-07T21:48:19.497678Z","iopub.execute_input":"2023-01-07T21:48:19.498132Z","iopub.status.idle":"2023-01-07T21:48:19.986596Z","shell.execute_reply.started":"2023-01-07T21:48:19.498098Z","shell.execute_reply":"2023-01-07T21:48:19.984842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}