{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qq git+https://github.com/PyFstat/PyFstat@python37\nimport os\nimport sys\nimport cv2\nimport h5py\nimport random\nimport math\nimport glob\nimport time\nimport multiprocessing\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport tensorflow as tf\n\nimport pyfstat\n\nfrom scipy import stats\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-17T13:53:28.070273Z","iopub.execute_input":"2022-12-17T13:53:28.070846Z","iopub.status.idle":"2022-12-17T13:53:41.436517Z","shell.execute_reply.started":"2022-12-17T13:53:28.070788Z","shell.execute_reply":"2022-12-17T13:53:41.435118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n\n# Target Height Equals ORiginal Height\nTARGET_HEIGHT = 360\nTARGET_HEIGHT_COMPRESSED = 360\nTARGET_HEIGHT_COMPRESS_RATIO = TARGET_HEIGHT // TARGET_HEIGHT_COMPRESSED\n\n# Width Is Compressed to 256 pixels\nTARGET_WIDTH = 4096\nTARGET_WIDTH_COMPRESSED = 512\nTARGET_WIDTH_COMPRESS_RATIO = TARGET_WIDTH // TARGET_WIDTH_COMPRESSED","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:53:41.439205Z","iopub.execute_input":"2022-12-17T13:53:41.43966Z","iopub.status.idle":"2022-12-17T13:53:41.447291Z","shell.execute_reply.started":"2022-12-17T13:53:41.439616Z","shell.execute_reply":"2022-12-17T13:53:41.446067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf test_data\n!mkdir -p test_data/x/H1\n!mkdir -p test_data/x/L1\n!ls -l test_data","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:53:41.448975Z","iopub.execute_input":"2022-12-17T13:53:41.449349Z","iopub.status.idle":"2022-12-17T13:53:45.839829Z","shell.execute_reply.started":"2022-12-17T13:53:41.449318Z","shell.execute_reply":"2022-12-17T13:53:45.838415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FILE_PATHS = glob.glob('/kaggle/input/g2net-detecting-continuous-gravitational-waves/test/*.hdf5')\nprint(f'Found {len(TRAIN_FILE_PATHS)} train samples')","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:53:45.843755Z","iopub.execute_input":"2022-12-17T13:53:45.845115Z","iopub.status.idle":"2022-12-17T13:53:45.881248Z","shell.execute_reply.started":"2022-12-17T13:53:45.845058Z","shell.execute_reply":"2022-12-17T13:53:45.88005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FILE_PATHS = sorted(TRAIN_FILE_PATHS)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:53:45.882733Z","iopub.execute_input":"2022-12-17T13:53:45.883669Z","iopub.status.idle":"2022-12-17T13:53:45.891053Z","shell.execute_reply.started":"2022-12-17T13:53:45.88362Z","shell.execute_reply":"2022-12-17T13:53:45.889721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_FILE_PATHS[0].split(\"/\")[-1].split(\".\")[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:53:45.892573Z","iopub.execute_input":"2022-12-17T13:53:45.893471Z","iopub.status.idle":"2022-12-17T13:53:45.905548Z","shell.execute_reply.started":"2022-12-17T13:53:45.893435Z","shell.execute_reply":"2022-12-17T13:53:45.904169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, path in enumerate(tqdm(TRAIN_FILE_PATHS)): \n    id = path.split(\"/\")[-1].split(\".\")[0]\n    \n    with h5py.File(path, 'r') as file:\n        SFT_H = np.array(file[id]['H1']['SFTs'], dtype=np.complex128)\n        SFT_L = np.array(file[id]['L1']['SFTs'], dtype=np.complex128)\n\n    signal_norm = {\n        'H1': SFT_H.real ** 2 + SFT_H.imag ** 2,\n        'L1': SFT_L.real ** 2 + SFT_L.imag ** 2,\n    }\n    \n    for detector, signal in signal_norm.items():\n#         patch = signal[:, :TARGET_WIDTH].reshape(TARGET_HEIGHT_COMPRESSED, TARGET_WIDTH_COMPRESSED,\n#                                                  TARGET_WIDTH_COMPRESS_RATIO).mean(axis=2)\n        patch = signal[:, :TARGET_WIDTH].reshape(TARGET_HEIGHT_COMPRESSED, TARGET_HEIGHT_COMPRESS_RATIO,\n                                                 TARGET_WIDTH_COMPRESSED, TARGET_WIDTH_COMPRESS_RATIO\n                                                ).mean(axis=(1,3))\n\n        # Save as PNG\n        patch_uint8 = (patch - patch.min())\n        patch_uint8 = patch_uint8 * (255 / patch_uint8.max())\n        patch_uint8 = patch_uint8.astype(np.uint8)\n        cv2.imwrite(f'test_data/x/{detector}/{i}.png', patch_uint8, [cv2.IMWRITE_PNG_COMPRESSION, 1])","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:53:45.907332Z","iopub.execute_input":"2022-12-17T13:53:45.90791Z","iopub.status.idle":"2022-12-17T13:56:45.698525Z","shell.execute_reply.started":"2022-12-17T13:53:45.907855Z","shell.execute_reply":"2022-12-17T13:56:45.696823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_VAL_SAMPLES_CREATED = len(glob.glob('test_data/x/*/*.png'))\nprint(f'N_VAL_SAMPLES_CREATED: {N_VAL_SAMPLES_CREATED}')","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:56:45.700697Z","iopub.status.idle":"2022-12-17T13:56:45.701173Z","shell.execute_reply.started":"2022-12-17T13:56:45.700946Z","shell.execute_reply":"2022-12-17T13:56:45.700967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -c --remove-files -f test_data.tar -C test_data x","metadata":{"execution":{"iopub.status.busy":"2022-12-17T13:56:45.702452Z","iopub.status.idle":"2022-12-17T13:56:45.702852Z","shell.execute_reply.started":"2022-12-17T13:56:45.702662Z","shell.execute_reply":"2022-12-17T13:56:45.702681Z"},"trusted":true},"execution_count":null,"outputs":[]}]}