{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# setup","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport torch as tc\nimport pandas as pd\nimport cv2,warnings,h5py\nfrom typing import Union\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-03T16:55:59.98536Z","iopub.execute_input":"2023-01-03T16:55:59.986641Z","iopub.status.idle":"2023-01-03T16:55:59.993714Z","shell.execute_reply.started":"2023-01-03T16:55:59.986583Z","shell.execute_reply":"2023-01-03T16:55:59.99238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def min_max_normalization(x:tc.Tensor)->tc.Tensor:\n    \"\"\"input.shape=(batch,f1,...)\"\"\"\n    shape=x.shape\n    if x.ndim>2:\n        x=x.reshape(x.shape[0],-1)\n    \n    min_=x.min(dim=-1,keepdim=True)[0]\n    max_=x.max(dim=-1,keepdim=True)[0]\n    if min_.mean()==0 and max_.mean()==1:\n        return x.reshape(shape)\n    \n    x=(x-min_)/(max_-min_+1e-9)\n    return x.reshape(shape)\n\ndef normalization(x:tc.Tensor)->tc.Tensor:\n    \"\"\"input.shape=(batch,f1,f2,...)\"\"\"\n    #[batch,f1,f2]->dim[1,2]\n    dim=list(range(1,x.ndim))\n    mean=x.mean(dim=dim,keepdim=True)\n    var=x.std(dim=dim,keepdim=True)\n    return (x-mean)/(var+1e-9)\n\ndef is_not_None(x)->bool:\n    \"\"\"return not (x is None or x is False)\"\"\"\n    return not (x is None or x is False)","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:55:59.996275Z","iopub.execute_input":"2023-01-03T16:55:59.996765Z","iopub.status.idle":"2023-01-03T16:56:00.008619Z","shell.execute_reply.started":"2023-01-03T16:55:59.996725Z","shell.execute_reply":"2023-01-03T16:56:00.007557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dataset","metadata":{}},{"cell_type":"code","source":"class Dataset_test:\n    \"\"\"\n    dataset = Dataset(data_type, df)\n\n    img, y = dataset[i]\n      img (np.float32): 2 x 360 x 128\n      y (np.float32): label 0 or 1\n    \"\"\"\n    def __init__(self):\n        super().__init__()\n        self.data_type = \"test\"\n        self.df = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, file_id):\n        #r = self.df.iloc[i]\n        img = np.empty((2, 360, 128), dtype=np.float32)\n\n        filename = '%s/%s/%s.hdf5' % (\"../input/g2net-detecting-continuous-gravitational-waves/\", self.data_type, file_id)\n        with h5py.File(filename, 'r') as f:\n            g = f[file_id]\n            \n            for ch, s in enumerate(['H1', 'L1']):\n                a = g[s]['SFTs'][:, :4096] * 1e22  # Fourier coefficient complex64\n                p = a.real**2 + a.imag**2  # power\n                p /= np.mean(p)  # normalize\n                p = np.mean(p.reshape(360, 128, 32), axis=2)  # compress 4096 -> 128\n                img[ch] = p\n\n        return tc.from_numpy(img)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-03T16:56:00.026542Z","iopub.execute_input":"2023-01-03T16:56:00.026867Z","iopub.status.idle":"2023-01-03T16:56:00.037735Z","shell.execute_reply.started":"2023-01-03T16:56:00.026839Z","shell.execute_reply":"2023-01-03T16:56:00.036578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generating gravitational waves from sine waves","metadata":{}},{"cell_type":"code","source":"def generate_signal(Graininess=1600,max_power_=(200,50),device=\"cuda:0\",r_=None,debug=False,\n                   data_augmentation=True):\n    \"\"\"return data:float->[0,1] shape=(2,360,128)\n            r_=[+float,+float]\"\"\"\n    ################################################hyperparameter\n    if not is_not_None(r_):\n        r_=[None,1]#zoom ratio\n        cache=np.random.random()\n        if cache<0.6:\n            r_[0]=np.random.randint(12,32+1)\n        if cache<0.8:\n            r_[0]=np.random.randint(1,12+1)\n        elif cache<1:\n            r_[0]=np.random.randint(1,4+1)\n            r_[1]=0.6-np.random.rand()*0.3#straighten\n    if debug:\n        print(f\"r_[0]:{r_[0]:.2f} ,r_[1]:{r_[1]:.2f}\")\n    \n    r_=tc.tensor(r_)\n    rate=np.random.random()*2+0.1\n    shape=tc.tensor([360,128])\n    \n    max_power=max_power_[1]+np.random.random()*(max_power_[0]-max_power_[1])\n    \n    ################################################generate sin signal\n    index=(tc.arange(Graininess,device=device)*2)/Graininess-1#[-1,1]\n    cache=np.random.random()*np.pi*2\n    \n    sin_data=tc.sin(index*rate+cache)#sin signal\n    sin_data=tc.stack((index,sin_data))\n    \n    ################################################\n    #Calculate the shortest distance from each point on the graph to sin to calculate the brightness of the point\n    x_=(tc.arange(128,device=device))/64-1\n    y_=(tc.arange(360,device=device))/180-1\n    data=1/(((sin_data[0][:,None,None]-x_[None,:,None])**2+(sin_data[1][:,None,None]-y_[None,None,:])**2).min(dim=0).values+1)\n    data:tc.Tensor\n    ################################################zoom\n    a=(shape//r_).to(tc.int).tolist()\n    #data=data[10:-10]\n    data=cv2.resize(data.T.cpu().numpy(),(a[1],a[0]))\n    data=data[:,:128]\n    ################################################Fine-tune graphics\n    \n    data=tc.from_numpy(data).to(device=device)\n    data=min_max_normalization(data[None])[0]\n    data=tc.pow(data,exponent=8)\n    data=tc.pow(data,exponent=8)\n    data=tc.pow(data,exponent=8)\n    if r_[0]<4:\n        max_power*=1.25\n    while data.sum()>max_power:\n        data=tc.pow(data,exponent=2)\n    if debug:\n        print(f\"power:{data.sum():.4f}\")\n    \n    ################################################padding\n    a=(-data.shape[0]+shape[0])//2+1\n    b=(-data.shape[1]+shape[1])//2\n\n    \n    a=tc.zeros(a,data.shape[1],device=device)\n    data=tc.cat((data,a),dim=0)\n    data=tc.cat((a,data),dim=0)\n    b=tc.zeros(data.shape[0],b,device=device)\n    data=tc.cat((data,b),dim=1)\n    data=tc.cat((b,data),dim=1)#shape=(360,128)\n    data=data.cpu()\n    \n    ################################################data augmentation\n    if data_augmentation:\n        data=np.roll(data.cpu().numpy(), np.random.randint(low=0, high=data.shape[0]), axis=0)\n        data=tc.from_numpy(data)[:360,:128]\n    data=normalization(data.T).T\n    \n    cache=min_max_normalization(data[None])[0]\n    ################################################\n    if cache.max(dim=1).values.sum()<2.5:#too flat\n        return generate_signal(Graininess=Graininess,max_power_=max_power_,device=device,debug=debug)\n    return data[None].repeat(2,1,1)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:56:00.039732Z","iopub.execute_input":"2023-01-03T16:56:00.040129Z","iopub.status.idle":"2023-01-03T16:56:00.062329Z","shell.execute_reply.started":"2023-01-03T16:56:00.040091Z","shell.execute_reply":"2023-01-03T16:56:00.061112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate gravitational waves is super fast","metadata":{}},{"cell_type":"code","source":"time=tqdm(total=10000)\nfor _ in range(10000):\n    data=generate_signal()\n    time.update()\ntime.close()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:56:00.064038Z","iopub.execute_input":"2023-01-03T16:56:00.06442Z","iopub.status.idle":"2023-01-03T16:56:15.537503Z","shell.execute_reply.started":"2023-01-03T16:56:00.064383Z","shell.execute_reply":"2023-01-03T16:56:15.535274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# show data","metadata":{}},{"cell_type":"code","source":"def show(data):\n    plt.subplot(121)\n    plt.imshow(data[0])\n    plt.subplot(122)\n    plt.imshow(data[1])\n    plt.show()\n\n    \n#We've found a lot of weird gravitational waves, and they're taking up a lot of frequencies\n#Many models don't detect them either\ndata=generate_signal(debug=True,data_augmentation=False,r_=[1,0.6])\nshow(data)\nprint('weird gravitational')\ndata=generate_signal(debug=True,data_augmentation=False,r_=[4,0.8])\nshow(data)\ndata=generate_signal(debug=True,data_augmentation=False,r_=[16,0.8])\nshow(data)\ndata=generate_signal(debug=True,data_augmentation=False,r_=[24,0.8])\nshow(data)\nprint('weird gravitational')\ndata=generate_signal(debug=True,data_augmentation=False,r_=[32,0.8])\nshow(data)\nprint(\"weird gravitational\")","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:56:15.538757Z","iopub.status.idle":"2023-01-03T16:56:15.539712Z","shell.execute_reply.started":"2023-01-03T16:56:15.539428Z","shell.execute_reply":"2023-01-03T16:56:15.539455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_data(x):\n    x=x.transpose(1,2)\n    x=x.reshape(256,360)\n    x=normalization(x).reshape(2,128,360)\n    x=x.transpose(1,2)\n    return x\n\ndataset=Dataset_test()\ndata=dataset[\"72c4060d6\"]\nshow(data)\n#This signal has interference, but as long as it is normalized, it will be fine.\ndata=fix_data(data)\n\nshow(data)\nprint(\"obviously there's a signal here\")\n","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:56:15.541322Z","iopub.status.idle":"2023-01-03T16:56:15.542799Z","shell.execute_reply.started":"2023-01-03T16:56:15.542484Z","shell.execute_reply":"2023-01-03T16:56:15.542522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_lb761=pd.read_csv(\"/kaggle/input/inception-v4-score-boost/submission.csv\")\nprint(sub_lb761[sub_lb761.id==\"72c4060d6\"])","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:56:15.544481Z","iopub.status.idle":"2023-01-03T16:56:15.545023Z","shell.execute_reply.started":"2023-01-03T16:56:15.544746Z","shell.execute_reply":"2023-01-03T16:56:15.544773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_all(x,str_=None):\n    plt.subplot(141)\n    plt.title(str_)\n    plt.imshow(x[0])\n    plt.subplot(142)\n    plt.imshow(x[1])\n\n    x=fix_data(x)\n    \n    plt.subplot(143)\n    plt.imshow(x[0])\n    plt.subplot(144)\n    plt.imshow(x[1])\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-03T16:56:15.546649Z","iopub.status.idle":"2023-01-03T16:56:15.547765Z","shell.execute_reply.started":"2023-01-03T16:56:15.547451Z","shell.execute_reply":"2023-01-03T16:56:15.547483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=dataset[\"20fac1e5a\"]\nstr_=f\"sub_lb761:{sub_lb761[sub_lb761.id=='20fac1e5a']['target'].to_numpy()[0]:.4f}\"\nshow_all(data,str_='weird gravitational'+str_)\ndata=dataset[\"70550b3be\"]\nstr_=f\"sub_lb761:{sub_lb761[sub_lb761.id=='70550b3be']['target'].to_numpy()[0]:.4f}\"\nshow_all(data,str_=str_)\ndata=dataset[\"c8a92cc43\"]\nstr_=f\"sub_lb761:{sub_lb761[sub_lb761.id=='c8a92cc43']['target'].to_numpy()[0]:.4f}\"\nshow_all(data,str_=str_)\ndata=dataset[\"e2256cda9\"]\nstr_=f\"sub_lb761:{sub_lb761[sub_lb761.id=='e2256cda9']['target'].to_numpy()[0]:.4f}\"\nshow_all(data,str_=str_)\n#These may all have gravitational waves\n#obviously i was guessing\n#so there may be some errors\n#This may cause errors when generating gravitational waves","metadata":{"execution":{"iopub.status.busy":"2023-01-03T16:56:15.549549Z","iopub.status.idle":"2023-01-03T16:56:15.550072Z","shell.execute_reply.started":"2023-01-03T16:56:15.5498Z","shell.execute_reply":"2023-01-03T16:56:15.549826Z"},"trusted":true},"execution_count":null,"outputs":[]}]}