{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os,shutil\nimport yaml\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport cv2\n\nimport torch\nfrom sklearn.model_selection import train_test_split\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainAfterNMSTable=pd.read_csv('../input/vinyolov5hyp/trainExcludeAndAfterNMS.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainAfterNMSTable.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainAfterNMSTable.loc[:,'image_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from sklearn.model_selection import StratifiedKFold\nskf = StratifiedKFold(n_splits=5,random_state=42,shuffle=True)\nlabels=trainAfterNMSTable['class_id']\nn=skf.get_n_splits(trainAfterNMSTable,labels)\ntrainIndex=[]\nvaliIndex=[]\nfor i,[train_index, test_index] in enumerate(skf.split(trainAfterNMSTable, labels)):\n    if i==0:\n        trainIndex=train_index\n        valiIndex=test_index\n#trainSet.info()\n#testSet.info()\ntrainSet=trainAfterNMSTable.iloc[trainIndex]\nvaliSet=trainAfterNMSTable.iloc[valiIndex]\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nskf = GroupKFold(n_splits=5)\nlabels=trainAfterNMSTable['class_id']\ngroups=trainAfterNMSTable['image_id']\nn=skf.get_n_splits(trainAfterNMSTable,labels,groups)\ntrainIndex=[]\nvaliIndex=[]\nfor i,[train_index, test_index] in enumerate(skf.split(trainAfterNMSTable, labels, groups)):\n    if i==2:\n        trainIndex=train_index\n        valiIndex=test_index\n#trainSet.info()\n#testSet.info()\ntrainSet=trainAfterNMSTable.iloc[trainIndex]\nvaliSet=trainAfterNMSTable.iloc[valiIndex]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainSet.loc[:,'class_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valiSet.loc[:,'class_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainSet.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for i in (11,0,13,3,7,8,9,10):\n    #trainSet=trainSet.append(valiSet[valiSet['class_id']==i])\n    #valiSet=valiSet.drop(valiSet[valiSet['class_id']==i].index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valiSet.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valiSet.loc[:,'class_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#i=0\n#j=0\n#for index,row in valiSet.iterrows():\n#    j=j+1\n#    if (row['image_id'] in trainSet['image_id'].values):\n#        valiSet=valiSet.append(trainSet[trainSet['image_id']==row['image_id']])\n#        trainSet=trainSet.drop(trainSet[trainSet['image_id']==row['image_id']].index)\n#        i=i+1\n#print(i,j)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#valiSet.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#valiSet.loc[:,'class_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#valiSet.loc[:,'class_id'].value_counts()/trainSet.loc[:,'class_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trainSet.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from sklearn.model_selection import StratifiedKFold\nos.makedirs('./vinforyolo/labels/train', exist_ok = True)\nos.makedirs('./vinforyolo/labels/val', exist_ok = True)\nos.makedirs('./vinforyolo/images/train', exist_ok = True)\nos.makedirs('./vinforyolo/images/val', exist_ok = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_LABELS_PATH = './vinforyolo/labels/train'\nVAL_LABELS_PATH = './vinforyolo/labels/val'\nTRAIN_IMAGES_PATH = './vinforyolo/images/train'\nVAL_IMAGES_PATH = './vinforyolo/images/val'\n\nfor index,row in tqdm(trainSet.iterrows()):\n    fileName=row['image_id']\n    boxInfo=str([int(row['class_id']),row['x_mid'],row['y_mid'],row['w'],row['h']])\n    boxInfo=boxInfo.replace(\",\",\"\")\n    boxInfo=boxInfo.replace(\"[\",\"\")\n    boxInfo=boxInfo.replace(\"]\",\"\")\n    file=open(os.path.join(TRAIN_LABELS_PATH,f\"{fileName}.txt\"), 'a')\n    #print(boxInfo)\n    file.write(boxInfo)\n    file.write('\\n')\n    file.close()\n    shutil.copy(\n        os.path.join('../input/vinbigdata-chest-xray-resized-png-1024x1024/train',f\"{fileName}.png\" ),          \n        TRAIN_IMAGES_PATH\n    )\n\n\nfor index,row in tqdm(valiSet.iterrows()):\n    fileName=row['image_id']\n    boxInfo=str([int(row['class_id']),row['x_mid'],row['y_mid'],row['w'],row['h']])\n    boxInfo=boxInfo.replace(\",\",\"\")\n    boxInfo=boxInfo.replace(\"[\",\"\")\n    boxInfo=boxInfo.replace(\"]\",\"\")\n    file=open(os.path.join(VAL_LABELS_PATH,f\"{fileName}.txt\"), 'a')\n    #print(boxInfo)\n    file.write(boxInfo)\n    file.write('\\n')\n    file.close()\n    shutil.copy(\n        os.path.join('../input/vinbigdata-chest-xray-resized-png-1024x1024/train',f\"{fileName}.png\" ),          \n        VAL_IMAGES_PATH\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_ids, class_names = list(zip(*set(zip(trainAfterNMSTable.class_id, trainAfterNMSTable.class_name))))\nclasses = list(np.array(class_names)[np.argsort(class_ids)])\nclasses = list(map(lambda x: str(x), classes))\nclasses","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nfrom glob import glob\nimport yaml\n\ncwd = './'\n\nwith open(join( cwd , 'train.txt'), 'w') as f:\n    for path in glob('./vinforyolo/images/train/*'):\n        f.write('.'+path+'\\n')\n            \nwith open(join( cwd , 'val.txt'), 'w') as f:\n    for path in glob('./vinforyolo/images/val/*'):\n        f.write('.'+path+'\\n')\n\ndata = dict(\n    train =  '../train.txt',\n    val   =  '../val.txt',\n    nc    = 14,\n    names = classes\n    )\n\nwith open(join( cwd , 'vinforyolo.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style=False)\n    \nf = open(join( cwd, 'vinforyolo.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.copytree('../input/yolov50326', '/kaggle/working/yolov5') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.copy('../input/vinyolov5xgroupkfold/general.py', '/kaggle/working/yolov5/utils') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('./yolov5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -r ./requirements.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python /kaggle/working/yolov5/train.py --img 1024 --batch 4 --epochs 15\\\n--data /kaggle/working/vinforyolo.yaml\\\n--cfg /kaggle/working/yolov5/models/hub/yolov5x6.yaml\\\n--weights yolov5x6.pt\\\n--hyp ../../input/vinyolov5hyp/yolov5xfromdis.yaml","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!WANDB_MODE=\"dryrun\" python /kaggle/working/yolov5/train.py --img 1024 --batch 4 --epochs 10\\\n--data /kaggle/working/vinforyolo.yaml\\\n--cfg /kaggle/working/yolov5/models/hub/yolov5x6.yaml\\\n--weights runs/train/exp/weights/last.pt\\\n--hyp ../../input/vinyolov5hyp/yolov5xfromdis.yaml","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a href=\"yolov5\"> Download File </a>"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"!WANDB_MODE=\"dryrun\" python /kaggle/working/yolov5/train.py --img 1024 --batch 4 --epochs 20\\\n--data /kaggle/working/vinforyolo.yaml\\\n--cfg /kaggle/working/yolov5/models/hub/yolov5x6.yaml\\\n--weights runs/train/exp/weights/last.pt\\\n--hyp ../../input/vinyolov5hyp/yolov5xfromdis.yaml"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"! find runs/train/exp |zip exp.zip -@"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"!WANDB_MODE=\"dryrun\" python /kaggle/working/yolov5/train.py --img 640 --batch 16 --epochs 20\\\n--data /kaggle/working/vinforyolo.yaml\\\n--cfg /kaggle/working/yolov5/models/yolov5x.yaml\\\n--weights ./runs/train/exp/weights/last.pt\\\n--hyp ../../input/vinyolov5hyp/yolov5xfromdis.yaml"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"! find runs/train/exp4 |zip exp4.zip -@"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"!WANDB_MODE=\"dryrun\" python /kaggle/working/yolov5/train.py --img 640 --batch 16 --epochs 20\\\n--data /kaggle/working/vinforyolo.yaml\\\n--cfg /kaggle/working/yolov5/models/yolov5x.yaml\\\n--weights ./runs/train/exp3/weights/last.pt\\\n--hyp ../../input/vinyolov5hyp/yolov5xfromdis.yaml"},{"metadata":{},"cell_type":"markdown","source":"<a href=\"yolov5\"> Download File </a>"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}