{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[],"dockerImageVersionId":30043,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!python --version","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:46:20.447328Z","iopub.execute_input":"2024-08-14T17:46:20.447659Z","iopub.status.idle":"2024-08-14T17:46:21.446768Z","shell.execute_reply.started":"2024-08-14T17:46:20.447623Z","shell.execute_reply":"2024-08-14T17:46:21.445841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow\nprint(tensorflow.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:46:21.987795Z","iopub.execute_input":"2024-08-14T17:46:21.988112Z","iopub.status.idle":"2024-08-14T17:46:26.522072Z","shell.execute_reply.started":"2024-08-14T17:46:21.988082Z","shell.execute_reply":"2024-08-14T17:46:26.521152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!git clone https://github.com/tensorflow/models.git\n\n%cd models/research/\n!git reset --hard 3f6fe2aa410d901aae8829597a65d084bffc20d3\n\n!protoc object_detection/protos/*.proto --python_out=.\n\n!cp object_detection/packages/tf2/setup.py .\n!python -m pip install . \n%cd /kaggle/working","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T17:46:26.523846Z","iopub.execute_input":"2024-08-14T17:46:26.524109Z","iopub.status.idle":"2024-08-14T17:47:44.40046Z","shell.execute_reply.started":"2024-08-14T17:46:26.524082Z","shell.execute_reply":"2024-08-14T17:47:44.399532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install roboflow\n\n# from roboflow import Roboflow\n\n# rf = Roboflow(api_key=\"rTmGDJBCgwOL8rnTDzQn\")\n# project = rf.workspace(\"arge-mfkgl\").project(\"fault-detection-nfpkz\")\n# version = project.version(2)\n# dataset = version.download(\"tensorflow\")\n\n# rf = Roboflow(api_key=\"rTmGDJBCgwOL8rnTDzQn\")\n# project = rf.workspace(\"arge-mfkgl\").project(\"fault-detection-nfpkz\")\n# version = project.version(2)\n# dataset = version.download(\"tfrecord\")\n\n\n!pip install roboflow\n\nfrom roboflow import Roboflow\n\n# rf = Roboflow(api_key=\"rTmGDJBCgwOL8rnTDzQn\")\n# project = rf.workspace(\"arge-mfkgl\").project(\"fault-detection-2-hyqsr\")\n# version = project.version(9)\n# dataset = version.download(\"tensorflow\")\n\n\n# rf = Roboflow(api_key=\"rTmGDJBCgwOL8rnTDzQn\")\n# project = rf.workspace(\"arge-mfkgl\").project(\"fault-detection-2-hyqsr\")\n# version = project.version(9)\n# dataset = version.download(\"tfrecord\")\n\n\n!pip install roboflow\nfrom roboflow import Roboflow\n\n\nrf = Roboflow(api_key=\"N59GoE3hEIBuhD3D7Bb1\")\nproject = rf.workspace(\"test-gjipz\").project(\"leke-m9bps\")\nversion = project.version(2)\ndataset = version.download(\"tensorflow\")\n\n\nrf = Roboflow(api_key=\"N59GoE3hEIBuhD3D7Bb1\")\nproject = rf.workspace(\"test-gjipz\").project(\"leke-m9bps\")\nversion = project.version(2)\ndataset = version.download(\"tfrecord\")","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:47:44.40219Z","iopub.execute_input":"2024-08-14T17:47:44.40251Z","iopub.status.idle":"2024-08-14T17:48:18.926161Z","shell.execute_reply.started":"2024-08-14T17:47:44.402477Z","shell.execute_reply":"2024-08-14T17:48:18.925269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:18.927822Z","iopub.execute_input":"2024-08-14T17:48:18.928099Z","iopub.status.idle":"2024-08-14T17:48:19.928977Z","shell.execute_reply.started":"2024-08-14T17:48:18.928071Z","shell.execute_reply":"2024-08-14T17:48:19.928124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:19.932354Z","iopub.execute_input":"2024-08-14T17:48:19.93264Z","iopub.status.idle":"2024-08-14T17:48:20.923894Z","shell.execute_reply.started":"2024-08-14T17:48:19.932613Z","shell.execute_reply":"2024-08-14T17:48:20.922104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = 'porselen-hata-bulma'\n#MODEL_PATH = 'efficientdet_d0_coco17_tpu-32'\nMODEL_PATH = 'efficientdet_d4_coco17_tpu-32'","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:20.928312Z","iopub.execute_input":"2024-08-14T17:48:20.928776Z","iopub.status.idle":"2024-08-14T17:48:20.934945Z","shell.execute_reply.started":"2024-08-14T17:48:20.92871Z","shell.execute_reply":"2024-08-14T17:48:20.933687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!rm -r {BASE_DIR}\n!mkdir {BASE_DIR}\n!mkdir {BASE_DIR}/pre-trained-models/\n!mkdir {BASE_DIR}/annotations\n!mkdir {BASE_DIR}/models\n!mkdir {BASE_DIR}/models/efficientdet/\n\n!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/{MODEL_PATH}.tar.gz\n\n!tar -xvzf {MODEL_PATH}.tar.gz\n!rm {MODEL_PATH}.tar.gz\n!mv {MODEL_PATH} {BASE_DIR}/pre-trained-models/\n!mv {BASE_DIR}/pre-trained-models/{MODEL_PATH}/pipeline.config {BASE_DIR}/models/efficientdet/pipeline.config","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T17:48:20.936842Z","iopub.execute_input":"2024-08-14T17:48:20.937261Z","iopub.status.idle":"2024-08-14T17:48:34.811162Z","shell.execute_reply.started":"2024-08-14T17:48:20.937222Z","shell.execute_reply":"2024-08-14T17:48:34.810011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib, cv2, os, time, functools\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport tensorflow as tf\n\nfrom google.protobuf import text_format","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T17:48:34.813613Z","iopub.execute_input":"2024-08-14T17:48:34.813962Z","iopub.status.idle":"2024-08-14T17:48:34.825878Z","shell.execute_reply.started":"2024-08-14T17:48:34.813915Z","shell.execute_reply":"2024-08-14T17:48:34.825005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from object_detection import inputs\n\nfrom object_detection.model_lib_v2 import eager_train_step\nfrom object_detection.model_lib_v2 import eager_eval_loop\nfrom object_detection.model_lib_v2 import load_fine_tune_checkpoint\nfrom object_detection.model_lib_v2 import get_filepath\nfrom object_detection.model_lib_v2 import clean_temporary_directories\n\nfrom object_detection.protos import pipeline_pb2\n\nfrom object_detection.utils import label_map_util\nfrom object_detection.utils import visualization_utils as viz_utils\nfrom object_detection.utils import config_util\n\nfrom object_detection.builders import dataset_builder\nfrom object_detection.builders import image_resizer_builder\nfrom object_detection.builders import model_builder\nfrom object_detection.builders import preprocessor_builder\n\nfrom object_detection.core import standard_fields as fields\n\nfrom object_detection.exporter_lib_v2 import DetectionInferenceModule","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:34.827586Z","iopub.execute_input":"2024-08-14T17:48:34.827948Z","iopub.status.idle":"2024-08-14T17:48:35.686698Z","shell.execute_reply.started":"2024-08-14T17:48:34.827916Z","shell.execute_reply":"2024-08-14T17:48:35.685898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $BASE_DIR","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:35.687994Z","iopub.execute_input":"2024-08-14T17:48:35.688282Z","iopub.status.idle":"2024-08-14T17:48:36.68241Z","shell.execute_reply.started":"2024-08-14T17:48:35.688252Z","shell.execute_reply":"2024-08-14T17:48:36.681502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al $BASE_DIR","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:36.684242Z","iopub.execute_input":"2024-08-14T17:48:36.684549Z","iopub.status.idle":"2024-08-14T17:48:37.675091Z","shell.execute_reply.started":"2024-08-14T17:48:36.68452Z","shell.execute_reply":"2024-08-14T17:48:37.674159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DIR = BASE_DIR + '/models/efficientdet/'\nPIPELINE_PATH = MODEL_DIR + 'pipeline.config'\nLABEL_MAP_PATH = BASE_DIR + '/annotations/label_map.pbtxt'\nOUTPUT_MODEL_DIR = '/kaggle/working/saved_model'","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:37.676933Z","iopub.execute_input":"2024-08-14T17:48:37.677236Z","iopub.status.idle":"2024-08-14T17:48:37.681809Z","shell.execute_reply.started":"2024-08-14T17:48:37.677208Z","shell.execute_reply":"2024-08-14T17:48:37.680822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/leke-2/train | grep tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:37.683116Z","iopub.execute_input":"2024-08-14T17:48:37.68338Z","iopub.status.idle":"2024-08-14T17:48:38.679691Z","shell.execute_reply.started":"2024-08-14T17:48:37.683354Z","shell.execute_reply":"2024-08-14T17:48:38.6787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/leke-2/valid | grep tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:38.681159Z","iopub.execute_input":"2024-08-14T17:48:38.681438Z","iopub.status.idle":"2024-08-14T17:48:39.675513Z","shell.execute_reply.started":"2024-08-14T17:48:38.68141Z","shell.execute_reply":"2024-08-14T17:48:39.674563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/working/leke-2/train/leke.tfrecord /kaggle/working/leke-2/train/fold_1.tfrecord\n!cp /kaggle/working/leke-2/valid/leke.tfrecord /kaggle/working/leke-2/valid/fold_1.tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:39.67734Z","iopub.execute_input":"2024-08-14T17:48:39.677777Z","iopub.status.idle":"2024-08-14T17:48:41.696944Z","shell.execute_reply.started":"2024-08-14T17:48:39.677714Z","shell.execute_reply":"2024-08-14T17:48:41.695907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/leke-2/train | grep tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:41.698546Z","iopub.execute_input":"2024-08-14T17:48:41.698838Z","iopub.status.idle":"2024-08-14T17:48:42.704859Z","shell.execute_reply.started":"2024-08-14T17:48:41.698809Z","shell.execute_reply":"2024-08-14T17:48:42.703944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/leke-2/valid | grep tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:42.706567Z","iopub.execute_input":"2024-08-14T17:48:42.706904Z","iopub.status.idle":"2024-08-14T17:48:43.78625Z","shell.execute_reply.started":"2024-08-14T17:48:42.706871Z","shell.execute_reply":"2024-08-14T17:48:43.785361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat /kaggle/working/leke-2/train/leke_label_map.pbtxt","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:43.787846Z","iopub.execute_input":"2024-08-14T17:48:43.788122Z","iopub.status.idle":"2024-08-14T17:48:44.785373Z","shell.execute_reply.started":"2024-08-14T17:48:43.788093Z","shell.execute_reply":"2024-08-14T17:48:44.784555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $LABEL_MAP_PATH","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:44.787192Z","iopub.execute_input":"2024-08-14T17:48:44.787624Z","iopub.status.idle":"2024-08-14T17:48:45.788574Z","shell.execute_reply.started":"2024-08-14T17:48:44.787584Z","shell.execute_reply":"2024-08-14T17:48:45.787679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#input_path = pathlib.Path('/kaggle/input/chest-xray-detection-512x512-groupkfold-tfrec')\n\ninput_path = pathlib.Path('/kaggle/working/leke-2/train')\n\n!cp {input_path}/leke_label_map.pbtxt {LABEL_MAP_PATH}\n\nDS_PATH = str(input_path)\nos.makedirs(OUTPUT_MODEL_DIR, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:45.790171Z","iopub.execute_input":"2024-08-14T17:48:45.790486Z","iopub.status.idle":"2024-08-14T17:48:46.7798Z","shell.execute_reply.started":"2024-08-14T17:48:45.790456Z","shell.execute_reply":"2024-08-14T17:48:46.778815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $DS_PATH","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:48:46.78122Z","iopub.execute_input":"2024-08-14T17:48:46.781504Z","iopub.status.idle":"2024-08-14T17:48:47.769644Z","shell.execute_reply.started":"2024-08-14T17:48:46.781476Z","shell.execute_reply":"2024-08-14T17:48:47.768774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['axes.grid'] = False\nplt.rcParams['xtick.labelsize'] = False\nplt.rcParams['ytick.labelsize'] = False\nplt.rcParams['xtick.top'] = False\nplt.rcParams['xtick.bottom'] = False\nplt.rcParams['ytick.left'] = False\nplt.rcParams['ytick.right'] = False\nplt.rcParams['figure.figsize'] = [12, 12]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T17:48:47.771452Z","iopub.execute_input":"2024-08-14T17:48:47.772027Z","iopub.status.idle":"2024-08-14T17:48:47.7785Z","shell.execute_reply.started":"2024-08-14T17:48:47.771974Z","shell.execute_reply":"2024-08-14T17:48:47.777666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 2020\nseed_everything(seed)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T17:48:47.779952Z","iopub.execute_input":"2024-08-14T17:48:47.78029Z","iopub.status.idle":"2024-08-14T17:48:47.789251Z","shell.execute_reply.started":"2024-08-14T17:48:47.780253Z","shell.execute_reply":"2024-08-14T17:48:47.788615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPU\")\n        strategy = tf.distribute.MirroredStrategy(devices=[\"GPU:0\"])\n    except RuntimeError as e:\n        gpu = None","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:06:59.518444Z","iopub.execute_input":"2024-08-14T18:06:59.518833Z","iopub.status.idle":"2024-08-14T18:06:59.528577Z","shell.execute_reply.started":"2024-08-14T18:06:59.518796Z","shell.execute_reply":"2024-08-14T18:06:59.527713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NUM_CLASSES = 14\nNUM_CLASSES = 1\nW = 1024\nH = 1024\n\nPER_REPLICA_BATCH_SIZE = 1\ntry:\n    REPLICAS = strategy.num_replicas_in_sync\nexcept:\n    REPLICAS = 1\n    \nBATCH_SIZE = PER_REPLICA_BATCH_SIZE * REPLICAS\n\nfold = 0\n\n#N_FOLDS = 5\nN_FOLDS = 1\n\nSCORE_THRESHOLD = 0.5","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-08-14T18:07:05.982032Z","iopub.execute_input":"2024-08-14T18:07:05.982386Z","iopub.status.idle":"2024-08-14T18:07:05.988443Z","shell.execute_reply.started":"2024-08-14T18:07:05.982355Z","shell.execute_reply":"2024-08-14T18:07:05.987465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"REPLICAS, BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:07.446185Z","iopub.execute_input":"2024-08-14T18:07:07.44652Z","iopub.status.idle":"2024-08-14T18:07:07.452326Z","shell.execute_reply.started":"2024-08-14T18:07:07.446491Z","shell.execute_reply":"2024-08-14T18:07:07.451512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $input_path ","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:08.507393Z","iopub.execute_input":"2024-08-14T18:07:08.507777Z","iopub.status.idle":"2024-08-14T18:07:09.603532Z","shell.execute_reply.started":"2024-08-14T18:07:08.50774Z","shell.execute_reply":"2024-08-14T18:07:09.602545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.min(train_df_eski['fold']), np.max(train_df_eski['fold'])  #tfrecord isimlerinin bir eksiği","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:09.605709Z","iopub.execute_input":"2024-08-14T18:07:09.606045Z","iopub.status.idle":"2024-08-14T18:07:09.609776Z","shell.execute_reply.started":"2024-08-14T18:07:09.606013Z","shell.execute_reply":"2024-08-14T18:07:09.608966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.min(train_df_eski['class_id']), np.max(train_df_eski['class_id'])  #tfrecord isimlerinin bir eksiği","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:10.106238Z","iopub.execute_input":"2024-08-14T18:07:10.106559Z","iopub.status.idle":"2024-08-14T18:07:10.110014Z","shell.execute_reply.started":"2024-08-14T18:07:10.106529Z","shell.execute_reply":"2024-08-14T18:07:10.10924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $input_path","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:10.51726Z","iopub.execute_input":"2024-08-14T18:07:10.517553Z","iopub.status.idle":"2024-08-14T18:07:11.612997Z","shell.execute_reply.started":"2024-08-14T18:07:10.517526Z","shell.execute_reply":"2024-08-14T18:07:11.612077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat /kaggle/working/leke-2/train/_annotations.csv | head","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:11.615679Z","iopub.execute_input":"2024-08-14T18:07:11.616109Z","iopub.status.idle":"2024-08-14T18:07:12.729376Z","shell.execute_reply.started":"2024-08-14T18:07:11.616066Z","shell.execute_reply":"2024-08-14T18:07:12.728114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(input_path / '_annotations.csv')\ntrain_df[\"fold\"] = 0\n\ntrain_df[\"class_name\"] = train_df[\"class\"]\ntrain_df[\"image_id\"] = train_df.index.astype(str)+\"_\"+train_df[\"filename\"]\n\n#train_df[\"class_id\"] = train_df['class_name'].apply(lambda x: 1 if x == 'paintstain' else 2)\ntrain_df[\"class_id\"] = 1\n\nprint(train_df.shape)\n\ntrain_df.head()\n\n# burada data içerisine 2 kolon eklememiz gerek. class_id ve fold. fold statik 0 olacak. 1 de rename olacak class => class_name","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:12.731102Z","iopub.execute_input":"2024-08-14T18:07:12.731403Z","iopub.status.idle":"2024-08-14T18:07:12.764091Z","shell.execute_reply.started":"2024-08-14T18:07:12.731372Z","shell.execute_reply":"2024-08-14T18:07:12.763358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/working/leke-2/valid/_annotations.csv')\ntest_df[\"fold\"] = 0\n\ntest_df[\"class_name\"] = test_df[\"class\"]\ntest_df[\"image_id\"] = test_df.index.astype(str)+\"_\"+test_df[\"filename\"]\n\n#test_df[\"class_id\"] = test_df['class_name'].apply(lambda x: 1 if x == 'paintstain' else 2)\ntest_df[\"class_id\"] = 1\n\nprint(test_df.shape)\n\ntest_df.head()\n\n# burada data içerisine 2 kolon eklememiz gerek. class_id ve fold. fold statik 0 olacak. 1 de rename olacak class => class_name","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:12.766088Z","iopub.execute_input":"2024-08-14T18:07:12.766371Z","iopub.status.idle":"2024-08-14T18:07:12.802213Z","shell.execute_reply.started":"2024-08-14T18:07:12.766342Z","shell.execute_reply":"2024-08-14T18:07:12.801088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.iloc[0][\"image_id\"], train_df.iloc[1][\"image_id\"]","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:12.80418Z","iopub.execute_input":"2024-08-14T18:07:12.804694Z","iopub.status.idle":"2024-08-14T18:07:12.814221Z","shell.execute_reply.started":"2024-08-14T18:07:12.804652Z","shell.execute_reply":"2024-08-14T18:07:12.812947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_index = label_map_util.create_category_index_from_labelmap(\n    LABEL_MAP_PATH,\n    use_display_name=True\n)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:12.81554Z","iopub.execute_input":"2024-08-14T18:07:12.816094Z","iopub.status.idle":"2024-08-14T18:07:12.822268Z","shell.execute_reply.started":"2024-08-14T18:07:12.816049Z","shell.execute_reply":"2024-08-14T18:07:12.821445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_index","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:13.16841Z","iopub.execute_input":"2024-08-14T18:07:13.168744Z","iopub.status.idle":"2024-08-14T18:07:13.174452Z","shell.execute_reply.started":"2024-08-14T18:07:13.168698Z","shell.execute_reply":"2024-08-14T18:07:13.173516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df['image_id'][train_df['fold'] != fold + 1].unique()), BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:13.618427Z","iopub.execute_input":"2024-08-14T18:07:13.618752Z","iopub.status.idle":"2024-08-14T18:07:13.625709Z","shell.execute_reply.started":"2024-08-14T18:07:13.618708Z","shell.execute_reply":"2024-08-14T18:07:13.624902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATASET = tf.io.gfile.glob(DS_PATH + f'/fold_{fold + 1}.tfrecord')\nTEST_DATASET = tf.io.gfile.glob(f'/kaggle/working/leke-2/valid/fold_{fold + 1}.tfrecord')    \n\nct_train = len(train_df['image_id'][train_df['fold'] != fold + 1].unique())  / BATCH_SIZE\nct_test = len(test_df['image_id'][test_df['fold'] == fold + 1].unique()) / BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:14.600172Z","iopub.execute_input":"2024-08-14T18:07:14.600501Z","iopub.status.idle":"2024-08-14T18:07:14.611587Z","shell.execute_reply.started":"2024-08-14T18:07:14.600471Z","shell.execute_reply":"2024-08-14T18:07:14.610687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATASET, TEST_DATASET","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:15.080421Z","iopub.execute_input":"2024-08-14T18:07:15.080741Z","iopub.status.idle":"2024-08-14T18:07:15.086174Z","shell.execute_reply.started":"2024-08-14T18:07:15.080697Z","shell.execute_reply":"2024-08-14T18:07:15.085091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_img_with_boxes(image, classes, boxes, scores=None, axis=None, plot=True):\n    if scores is None:\n        scores = np.ones(len(classes))\n        \n    image_with_detections = image.copy()\n    \n    viz_utils.visualize_boxes_and_labels_on_image_array(\n          image_with_detections,\n          boxes,\n          classes,\n          scores,\n          category_index,\n          use_normalized_coordinates=True,\n          max_boxes_to_draw=100,\n          min_score_thresh=SCORE_THRESHOLD,\n          agnostic_mode=False)\n    \n    if plot:\n        if axis is None:\n            plt.figure(figsize=(12,12))\n            plt.imshow(image_with_detections)\n            plt.show()\n        else:\n            axis.imshow(image_with_detections)\n    else:\n        return image_with_detections","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:07:15.63781Z","iopub.execute_input":"2024-08-14T18:07:15.638139Z","iopub.status.idle":"2024-08-14T18:07:15.646657Z","shell.execute_reply.started":"2024-08-14T18:07:15.638106Z","shell.execute_reply":"2024-08-14T18:07:15.645917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_description = {\n    'image/height': tf.io.FixedLenFeature([], tf.int64),\n    'image/width': tf.io.FixedLenFeature([], tf.int64),\n    'image/filename': tf.io.FixedLenFeature([], tf.string),\n    #'image/source_id': tf.io.FixedLenFeature([], tf.string),\n    'image/encoded': tf.io.FixedLenFeature([], tf.string),\n    'image/format': tf.io.FixedLenFeature([], tf.string),\n    'image/object/bbox/xmin': tf.io.FixedLenSequenceFeature([], tf.float32, True),\n    'image/object/bbox/xmax': tf.io.FixedLenSequenceFeature([], tf.float32, True),\n    'image/object/bbox/ymin': tf.io.FixedLenSequenceFeature([], tf.float32, True),\n    'image/object/bbox/ymax': tf.io.FixedLenSequenceFeature([], tf.float32, True),\n    'image/object/class/text': tf.io.FixedLenSequenceFeature([], tf.string, True),\n    'image/object/class/label': tf.io.FixedLenSequenceFeature([], tf.int64, True)\n}\n\ndef parse_image_sample(example_proto):\n    return tf.io.parse_single_example(example_proto,\n                                      feature_description)\n\nraw_image_dataset = tf.data.TFRecordDataset(TEST_DATASET)\nparsed_image_dataset = raw_image_dataset.map(parse_image_sample)\niterator = iter(parsed_image_dataset)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:07:16.380838Z","iopub.execute_input":"2024-08-14T18:07:16.381157Z","iopub.status.idle":"2024-08-14T18:07:16.441265Z","shell.execute_reply.started":"2024-08-14T18:07:16.381128Z","shell.execute_reply":"2024-08-14T18:07:16.440594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DATASET[0]","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:16.844935Z","iopub.execute_input":"2024-08-14T18:07:16.845251Z","iopub.status.idle":"2024-08-14T18:07:16.850167Z","shell.execute_reply.started":"2024-08-14T18:07:16.845223Z","shell.execute_reply":"2024-08-14T18:07:16.849169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!sha1sum /kaggle/working/leke-2/valid/fold_1.tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:17.231187Z","iopub.execute_input":"2024-08-14T18:07:17.231504Z","iopub.status.idle":"2024-08-14T18:07:18.332014Z","shell.execute_reply.started":"2024-08-14T18:07:17.231472Z","shell.execute_reply":"2024-08-14T18:07:18.331089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_image_dataset","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:18.33452Z","iopub.execute_input":"2024-08-14T18:07:18.334844Z","iopub.status.idle":"2024-08-14T18:07:18.340478Z","shell.execute_reply.started":"2024-08-14T18:07:18.334811Z","shell.execute_reply":"2024-08-14T18:07:18.339717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parsed_image_dataset","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:18.341663Z","iopub.execute_input":"2024-08-14T18:07:18.342038Z","iopub.status.idle":"2024-08-14T18:07:18.352111Z","shell.execute_reply.started":"2024-08-14T18:07:18.342009Z","shell.execute_reply":"2024-08-14T18:07:18.35123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iterator","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:18.827828Z","iopub.execute_input":"2024-08-14T18:07:18.828128Z","iopub.status.idle":"2024-08-14T18:07:18.833567Z","shell.execute_reply.started":"2024-08-14T18:07:18.8281Z","shell.execute_reply":"2024-08-14T18:07:18.83256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = next(iterator)\n\nprint(temp.keys())\nprint(temp['image/height'], temp['image/width'])\n\nplt.imshow(tf.io.decode_jpeg(temp['image/encoded']).numpy())\nplt.axis('off')  # Eksenleri kapatmak isterseniz\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:07:19.527407Z","iopub.execute_input":"2024-08-14T18:07:19.527711Z","iopub.status.idle":"2024-08-14T18:07:19.816507Z","shell.execute_reply.started":"2024-08-14T18:07:19.527681Z","shell.execute_reply":"2024-08-14T18:07:19.815739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs = None\n\nconfigs = config_util.get_configs_from_pipeline_file(PIPELINE_PATH)\n\nconfigs['model'].ssd.num_classes = NUM_CLASSES\n\nconfigs['train_config'].sync_replicas = True if REPLICAS > 1 else False\nconfigs['train_config'].replicas_to_aggregate = REPLICAS\nconfigs['train_config'].batch_size = BATCH_SIZE\n\nPIPELINE_PATH.replace('pipeline.config', '')\nconfigs['train_config'].fine_tune_checkpoint = (\n    BASE_DIR + f'/pre-trained-models/{MODEL_PATH}/checkpoint/ckpt-0'\n)\nconfigs['train_config'].fine_tune_checkpoint_type = \"detection\"\n\nconfigs['train_input_config'].label_map_path = LABEL_MAP_PATH\nconfigs['train_input_config'].tf_record_input_reader.input_path[:] = TRAIN_DATASET\nconfigs['train_input_config'].load_multiclass_scores = True\n\nconfigs['eval_config'].batch_size = 1\nconfigs['eval_config'].metrics_set[:] = ''\nconfigs['eval_config'].metrics_set.append('pascal_voc_detection_metrics')\n\nconfigs['eval_input_config'].label_map_path = LABEL_MAP_PATH\nconfigs['eval_input_config'].tf_record_input_reader.input_path[:] = TEST_DATASET\nconfigs['eval_input_config'].load_multiclass_scores = True\n\nconfig_util.save_pipeline_config(config_util.create_pipeline_proto_from_configs(configs), PIPELINE_PATH.replace('pipeline.config', ''))","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:08:58.487127Z","iopub.execute_input":"2024-08-14T18:08:58.487531Z","iopub.status.idle":"2024-08-14T18:08:58.50954Z","shell.execute_reply.started":"2024-08-14T18:08:58.487495Z","shell.execute_reply":"2024-08-14T18:08:58.508676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs['train_config'].data_augmentation_options","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:05.470509Z","iopub.execute_input":"2024-08-14T18:09:05.470865Z","iopub.status.idle":"2024-08-14T18:09:05.476438Z","shell.execute_reply.started":"2024-08-14T18:09:05.470833Z","shell.execute_reply":"2024-08-14T18:09:05.475529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = configs['train_config'].data_augmentation_options.pop()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:28.750113Z","iopub.execute_input":"2024-08-14T18:09:28.75053Z","iopub.status.idle":"2024-08-14T18:09:28.754532Z","shell.execute_reply.started":"2024-08-14T18:09:28.750494Z","shell.execute_reply":"2024-08-14T18:09:28.753601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs['train_config'].data_augmentation_options","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:34.066753Z","iopub.execute_input":"2024-08-14T18:09:34.067126Z","iopub.status.idle":"2024-08-14T18:09:34.073545Z","shell.execute_reply.started":"2024-08-14T18:09:34.067093Z","shell.execute_reply":"2024-08-14T18:09:34.072576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATASET, TEST_DATASET, LABEL_MAP_PATH, PIPELINE_PATH, PIPELINE_PATH.replace('pipeline.config', '')","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:38.825688Z","iopub.execute_input":"2024-08-14T18:09:38.826057Z","iopub.status.idle":"2024-08-14T18:09:38.832659Z","shell.execute_reply.started":"2024-08-14T18:09:38.826023Z","shell.execute_reply":"2024-08-14T18:09:38.831576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs.keys()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:39.728073Z","iopub.execute_input":"2024-08-14T18:09:39.728463Z","iopub.status.idle":"2024-08-14T18:09:39.733998Z","shell.execute_reply.started":"2024-08-14T18:09:39.728416Z","shell.execute_reply":"2024-08-14T18:09:39.733237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs[\"train_input_config\"]","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:40.673028Z","iopub.execute_input":"2024-08-14T18:09:40.673416Z","iopub.status.idle":"2024-08-14T18:09:40.681314Z","shell.execute_reply.started":"2024-08-14T18:09:40.673376Z","shell.execute_reply":"2024-08-14T18:09:40.679811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat porselen-hata-bulma/annotations/label_map.pbtxt","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:41.597783Z","iopub.execute_input":"2024-08-14T18:09:41.598131Z","iopub.status.idle":"2024-08-14T18:09:42.707511Z","shell.execute_reply.started":"2024-08-14T18:09:41.598102Z","shell.execute_reply":"2024-08-14T18:09:42.706471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!sha1sum /kaggle/working/leke-2/train/fold_1.tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:42.710054Z","iopub.execute_input":"2024-08-14T18:09:42.710373Z","iopub.status.idle":"2024-08-14T18:09:43.831264Z","shell.execute_reply.started":"2024-08-14T18:09:42.710341Z","shell.execute_reply":"2024-08-14T18:09:43.829931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_config = configs['model']\n\ntrain_config = configs['train_config']\ntrain_input_config = configs['train_input_config']\n\neval_config = configs['eval_config']\neval_input_config = configs['eval_input_configs'][0]","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:43.833598Z","iopub.execute_input":"2024-08-14T18:09:43.833952Z","iopub.status.idle":"2024-08-14T18:09:43.839479Z","shell.execute_reply.started":"2024-08-14T18:09:43.833917Z","shell.execute_reply":"2024-08-14T18:09:43.838583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_input_config","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:44.205875Z","iopub.execute_input":"2024-08-14T18:09:44.206232Z","iopub.status.idle":"2024-08-14T18:09:44.212596Z","shell.execute_reply.started":"2024-08-14T18:09:44.206201Z","shell.execute_reply":"2024-08-14T18:09:44.211675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_input_config","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:44.818957Z","iopub.execute_input":"2024-08-14T18:09:44.81932Z","iopub.status.idle":"2024-08-14T18:09:44.824601Z","shell.execute_reply.started":"2024-08-14T18:09:44.819291Z","shell.execute_reply":"2024-08-14T18:09:44.823809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_fn(inputs):\n    return inputs / 255.0\n\ndef build_model():\n    detection_model = model_builder._build_ssd_model(ssd_config=model_config.ssd, is_training=True, add_summaries=False)\n\n    detection_model._feature_extractor.preprocess = preprocess_fn\n    \n    return detection_model\n\ntry:\n    with strategy.scope():\n        detection_model = build_model()\nexcept:\n    detection_model = build_model()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:45.62875Z","iopub.execute_input":"2024-08-14T18:09:45.629124Z","iopub.status.idle":"2024-08-14T18:09:53.362179Z","shell.execute_reply.started":"2024-08-14T18:09:45.629088Z","shell.execute_reply":"2024-08-14T18:09:53.361426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SHAPE = (H, W)\nlearning_rate = 1e-5\n\nEPOCHS = 10\nSTEPS_PER_EPOCH = int(ct_train)\nNUM_TRAIN_STEPS = int(STEPS_PER_EPOCH * EPOCHS)\ntrain_steps = NUM_TRAIN_STEPS\n\nRUN_EVAL = True\nMONITOR_METRIC = 'PascalBoxes_Precision/mAP@0.5IOU'\nES_PATIENCE = 5\n\nbest_metric_value = 0.0\nnot_improved = 0\nsteps_per_sec_list = []\n\nunpad_groundtruth_tensors = train_config.unpad_groundtruth_tensors\nadd_regularization_loss = train_config.add_regularization_loss\n\nclip_gradients_value = None\nif train_config.gradient_clipping_by_norm > 0:\n    clip_gradients_value = train_config.gradient_clipping_by_norm\n\nconfig_util.update_fine_tune_checkpoint_type(train_config)\nfine_tune_checkpoint_type = train_config.fine_tune_checkpoint_type\nfine_tune_checkpoint_version = train_config.fine_tune_checkpoint_version","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:09:53.364522Z","iopub.execute_input":"2024-08-14T18:09:53.364951Z","iopub.status.idle":"2024-08-14T18:09:53.374015Z","shell.execute_reply.started":"2024-08-14T18:09:53.364907Z","shell.execute_reply":"2024-08-14T18:09:53.372959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_dataset_fn(input_context):\n    def transform_input_data_fn(tensor_dict):\n        data_augmentation_options = [\n            preprocessor_builder.build(step)\n            for step in train_config.data_augmentation_options\n        ]\n        data_augmentation_fn = functools.partial(\n            inputs.augment_input_data,\n            data_augmentation_options=data_augmentation_options\n        )\n\n        image_resizer_config = model_config.ssd.image_resizer\n        image_resizer_fn = image_resizer_builder.build(image_resizer_config)\n        transform_data_fn = functools.partial(\n            inputs.transform_input_data, \n            model_preprocess_fn=detection_model.preprocess,\n            image_resizer_fn=image_resizer_fn,\n            num_classes=NUM_CLASSES,\n            data_augmentation_fn=data_augmentation_fn,\n            merge_multiple_boxes=False,\n            use_multiclass_scores=False\n        )\n\n        tensor_dict = inputs.pad_input_data_to_static_shapes(\n            tensor_dict=transform_data_fn(tensor_dict),\n            max_num_boxes=train_input_config.max_number_of_boxes,\n            num_classes=NUM_CLASSES,\n            spatial_image_shape=SHAPE\n        )\n\n        return (inputs._get_features_dict(tensor_dict, False),\n                inputs._get_labels_dict(tensor_dict))\n    \n    train_input = dataset_builder.build(\n        train_input_config,\n        transform_input_data_fn=transform_input_data_fn,\n        batch_size=train_config.batch_size,\n        input_context=input_context,\n    )\n    train_input = train_input.repeat()    \n\n    return train_input","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-08-14T18:09:53.375352Z","iopub.execute_input":"2024-08-14T18:09:53.375703Z","iopub.status.idle":"2024-08-14T18:09:53.387564Z","shell.execute_reply.started":"2024-08-14T18:09:53.375663Z","shell.execute_reply":"2024-08-14T18:09:53.386798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_dataset_fn(input_context):\n    def transform_input_data_fn(tensor_dict):\n        image_resizer_config = model_config.ssd.image_resizer\n        image_resizer_fn = image_resizer_builder.build(image_resizer_config)\n\n        transform_data_fn = functools.partial(\n            inputs.transform_input_data, \n            model_preprocess_fn=detection_model.preprocess,\n            image_resizer_fn=image_resizer_fn,\n            num_classes=NUM_CLASSES,\n            merge_multiple_boxes=False,\n            use_multiclass_scores=False,\n            retain_original_image=eval_config.retain_original_images,\n            retain_original_image_additional_channels=eval_config.retain_original_image_additional_channels\n        )\n\n        tensor_dict = inputs.pad_input_data_to_static_shapes(\n            tensor_dict=transform_data_fn(tensor_dict),\n            max_num_boxes=eval_input_config.max_number_of_boxes,\n            num_classes=NUM_CLASSES,\n            spatial_image_shape=SHAPE\n        )\n\n        return (inputs._get_features_dict(tensor_dict, False),\n                inputs._get_labels_dict(tensor_dict))\n\n    eval_input = dataset_builder.build(\n        eval_input_config,\n        transform_input_data_fn=transform_input_data_fn,\n        batch_size=eval_config.batch_size,\n        input_context=input_context,\n    )\n\n    return eval_input","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-08-14T18:09:53.388856Z","iopub.execute_input":"2024-08-14T18:09:53.389158Z","iopub.status.idle":"2024-08-14T18:09:53.399843Z","shell.execute_reply.started":"2024-08-14T18:09:53.389128Z","shell.execute_reply":"2024-08-14T18:09:53.398996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_input = strategy.experimental_distribute_datasets_from_function(train_dataset_fn)\ntrain_input_iter = iter(train_input)\n\neval_input = strategy.experimental_distribute_datasets_from_function(eval_dataset_fn)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-08-14T18:09:53.402312Z","iopub.execute_input":"2024-08-14T18:09:53.402714Z","iopub.status.idle":"2024-08-14T18:09:55.522293Z","shell.execute_reply.started":"2024-08-14T18:09:53.402667Z","shell.execute_reply":"2024-08-14T18:09:55.521467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n\nN = 4\nfig, ax = plt.subplots(int(np.sqrt(N)), int(np.sqrt(N)), figsize=(12,12))\nax = ax.flatten()\n\ntrain_dataset = train_dataset_fn(None).unbatch().batch(1)\ntrain_iter = iter(train_dataset)\n\nfor idx in range(N):\n    features, labels = next(train_iter)\n    \n    print(features['hash'])\n    \n    image = features['image'][0].numpy()    \n    \n    n_boxes = labels['num_groundtruth_boxes'][0].numpy()\n    boxes = labels['groundtruth_boxes'][0, :n_boxes, :].numpy()\n    classes = labels['groundtruth_classes'][0, :n_boxes, :].numpy()\n    classes = np.argmax(classes, axis=-1) + 1\n    \n    plot_img_with_boxes((image*255).astype('uint8'), \n                        classes, \n                        boxes,\n                        axis=ax[idx])\n    \nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:09:55.523693Z","iopub.execute_input":"2024-08-14T18:09:55.523983Z","iopub.status.idle":"2024-08-14T18:09:58.508924Z","shell.execute_reply.started":"2024-08-14T18:09:55.523955Z","shell.execute_reply":"2024-08-14T18:09:58.508149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n\nN = 4\nfig, ax = plt.subplots(int(np.sqrt(N)), int(np.sqrt(N)), figsize=(12,12))\nax = ax.flatten()\n\neval_dataset = eval_dataset_fn(None).unbatch().batch(1)\neval_iter = iter(eval_dataset)\n\nfor idx in range(N):\n    features, labels = next(eval_iter)\n    image = features['image'][0].numpy()\n    n_boxes = labels['num_groundtruth_boxes'][0].numpy()\n    boxes = labels['groundtruth_boxes'][0, :n_boxes, :].numpy()\n    classes = labels['groundtruth_classes'][0, :n_boxes, :].numpy()\n    classes = np.argmax(classes, axis=-1) + 1\n    \n    plot_img_with_boxes((image*255).astype('uint8'), \n                        classes, \n                        boxes,\n                        axis=ax[idx])\n    \nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:10:06.461319Z","iopub.execute_input":"2024-08-14T18:10:06.461754Z","iopub.status.idle":"2024-08-14T18:10:09.497584Z","shell.execute_reply.started":"2024-08-14T18:10:06.461701Z","shell.execute_reply":"2024-08-14T18:10:09.49681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    global_step = tf.Variable(0,\n                              trainable=False,\n                              dtype=tf.compat.v2.dtypes.int64,\n                              name='global_step',\n                              aggregation=tf.compat.v2.VariableAggregation.ONLY_FIRST_REPLICA)\n    \n    checkpointed_step = int(global_step.value())\n    logged_step = int(global_step.value())\n    total_loss = 0\n\n    if train_config.fine_tune_checkpoint:\n        load_fine_tune_checkpoint(detection_model,\n                                  train_config.fine_tune_checkpoint,\n                                  fine_tune_checkpoint_type,\n                                  fine_tune_checkpoint_version,\n                                  train_input,\n                                  unpad_groundtruth_tensors)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:10:13.583241Z","iopub.execute_input":"2024-08-14T18:10:13.583622Z","iopub.status.idle":"2024-08-14T18:10:57.113798Z","shell.execute_reply.started":"2024-08-14T18:10:13.583582Z","shell.execute_reply":"2024-08-14T18:10:57.113014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_DIR","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:05.909955Z","iopub.execute_input":"2024-08-14T18:11:05.910307Z","iopub.status.idle":"2024-08-14T18:11:05.915603Z","shell.execute_reply.started":"2024-08-14T18:11:05.910276Z","shell.execute_reply":"2024-08-14T18:11:05.914682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al $MODEL_DIR","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:06.517326Z","iopub.execute_input":"2024-08-14T18:11:06.517671Z","iopub.status.idle":"2024-08-14T18:11:07.629502Z","shell.execute_reply.started":"2024-08-14T18:11:06.517641Z","shell.execute_reply":"2024-08-14T18:11:07.628558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!chmod 777 -R $MODEL_DIR","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:07.63195Z","iopub.execute_input":"2024-08-14T18:11:07.632344Z","iopub.status.idle":"2024-08-14T18:11:08.740794Z","shell.execute_reply.started":"2024-08-14T18:11:07.632301Z","shell.execute_reply":"2024-08-14T18:11:08.739587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    if callable(learning_rate):\n        learning_rate_fn = learning_rate\n    else:\n        learning_rate_fn = lambda: learning_rate\n\n    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n    \n    ckpt = tf.compat.v2.train.Checkpoint(step=global_step,\n                                         model=detection_model,\n                                         optimizer=optimizer)\n\n    manager_dir = get_filepath(strategy, MODEL_DIR)\n    \n    manager = tf.compat.v2.train.CheckpointManager(ckpt,\n                                                   manager_dir,\n                                                   max_to_keep=1)\n\n    latest_checkpoint = tf.train.latest_checkpoint(MODEL_DIR)\n    ckpt.restore(latest_checkpoint).expect_partial()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:11:08.74342Z","iopub.execute_input":"2024-08-14T18:11:08.743859Z","iopub.status.idle":"2024-08-14T18:11:08.756357Z","shell.execute_reply.started":"2024-08-14T18:11:08.743813Z","shell.execute_reply":"2024-08-14T18:11:08.755566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ckpt","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:09.031482Z","iopub.execute_input":"2024-08-14T18:11:09.031803Z","iopub.status.idle":"2024-08-14T18:11:09.036664Z","shell.execute_reply.started":"2024-08-14T18:11:09.031771Z","shell.execute_reply":"2024-08-14T18:11:09.035939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"manager_dir","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:09.67767Z","iopub.execute_input":"2024-08-14T18:11:09.678006Z","iopub.status.idle":"2024-08-14T18:11:09.683129Z","shell.execute_reply.started":"2024-08-14T18:11:09.677976Z","shell.execute_reply":"2024-08-14T18:11:09.68235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"manager","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:10.724304Z","iopub.execute_input":"2024-08-14T18:11:10.724674Z","iopub.status.idle":"2024-08-14T18:11:10.731382Z","shell.execute_reply.started":"2024-08-14T18:11:10.724621Z","shell.execute_reply":"2024-08-14T18:11:10.73036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"latest_checkpoint","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:11.760398Z","iopub.execute_input":"2024-08-14T18:11:11.760805Z","iopub.status.idle":"2024-08-14T18:11:11.764421Z","shell.execute_reply.started":"2024-08-14T18:11:11.760763Z","shell.execute_reply":"2024-08-14T18:11:11.763523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    def train_step_fn(features, labels):\n        loss = eager_train_step(detection_model,\n                                features,\n                                labels,\n                                unpad_groundtruth_tensors,\n                                optimizer,\n                                learning_rate=learning_rate_fn(),\n                                add_regularization_loss=add_regularization_loss,\n                                clip_gradients_value=clip_gradients_value,\n                                global_step=global_step,\n                                num_replicas=REPLICAS)\n        global_step.assign_add(1)\n        return loss\n\n    def _sample_and_train(strategy, train_step_fn, data_iterator):\n        features, labels = data_iterator.next()\n        per_replica_losses = strategy.run(train_step_fn, \n                                          args=(features, labels))\n        return strategy.reduce(tf.distribute.ReduceOp.SUM,\n                               per_replica_losses, axis=None)\n\n    @tf.function\n    def _dist_train_step(data_iterator):\n        return _sample_and_train(strategy, \n                                 train_step_fn, \n                                 data_iterator)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T18:11:12.83629Z","iopub.execute_input":"2024-08-14T18:11:12.836647Z","iopub.status.idle":"2024-08-14T18:11:12.847311Z","shell.execute_reply.started":"2024-08-14T18:11:12.836616Z","shell.execute_reply":"2024-08-14T18:11:12.846323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_steps, STEPS_PER_EPOCH","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:11:14.063285Z","iopub.execute_input":"2024-08-14T18:11:14.063641Z","iopub.status.idle":"2024-08-14T18:11:14.069416Z","shell.execute_reply.started":"2024-08-14T18:11:14.063609Z","shell.execute_reply":"2024-08-14T18:11:14.068436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 20\nSTEPS_PER_EPOCH = int(ct_train)\nNUM_TRAIN_STEPS = int(STEPS_PER_EPOCH * EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:49:03.188609Z","iopub.execute_input":"2024-08-14T18:49:03.189015Z","iopub.status.idle":"2024-08-14T18:49:03.193293Z","shell.execute_reply.started":"2024-08-14T18:49:03.188977Z","shell.execute_reply":"2024-08-14T18:49:03.192453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN_STEPS","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:49:03.956179Z","iopub.execute_input":"2024-08-14T18:49:03.956487Z","iopub.status.idle":"2024-08-14T18:49:03.961632Z","shell.execute_reply.started":"2024-08-14T18:49:03.956459Z","shell.execute_reply":"2024-08-14T18:49:03.960721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_steps += NUM_TRAIN_STEPS","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:49:05.02713Z","iopub.execute_input":"2024-08-14T18:49:05.027462Z","iopub.status.idle":"2024-08-14T18:49:05.031677Z","shell.execute_reply.started":"2024-08-14T18:49:05.027432Z","shell.execute_reply":"2024-08-14T18:49:05.030664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"range(global_step.value(), train_steps)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:49:06.6527Z","iopub.execute_input":"2024-08-14T18:49:06.653108Z","iopub.status.idle":"2024-08-14T18:49:06.659103Z","shell.execute_reply.started":"2024-08-14T18:49:06.65307Z","shell.execute_reply":"2024-08-14T18:49:06.658178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last_step_time = time.time()\n\nfor _ in range(global_step.value(), train_steps):\n    with strategy.scope():\n        loss = _dist_train_step(train_input_iter)\n        time_taken = time.time() - last_step_time\n        last_step_time = time.time()\n        steps_per_sec = 1.0 / time_taken\n        steps_per_sec_list.append(steps_per_sec)\n        total_loss += loss\n        \n    if int(global_step.value()) % STEPS_PER_EPOCH == 0:\n        if not RUN_EVAL:\n            print('Epoch {} [ETA {:.2f}s] loss={:.3f}'.format(\n                  int(global_step.value()) // STEPS_PER_EPOCH,\n                  time_taken * STEPS_PER_EPOCH,\n                  total_loss / STEPS_PER_EPOCH))\n        else:\n            eval_global_step = tf.compat.v2.Variable(0, \n                                                     trainable=False,\n                                                     dtype=tf.compat.v2.dtypes.int64)\n\n            eval_metrics = eager_eval_loop(detection_model,\n                                           configs,\n                                           eval_input,\n                                           global_step=eval_global_step)\n\n            print('Epoch {} [ETA {:.2f}s] loss={:.3f} mAP@.5={:.3f}'.format(\n                  int(global_step.value()) // STEPS_PER_EPOCH,\n                  time_taken * STEPS_PER_EPOCH,\n                  total_loss / STEPS_PER_EPOCH,\n                  eval_metrics[MONITOR_METRIC]))\n\n#             if eval_metrics[MONITOR_METRIC] > best_metric_value:\n#                 best_metric_value = eval_metrics[MONITOR_METRIC]\n#                 manager.save()\n#                 not_improved = 0\n#             else:\n#                 not_improved += 1\n\n#             if not_improved >= ES_PATIENCE:\n#                 print(f\"Early stopping at epoch {int(global_step.value()) // STEPS_PER_EPOCH}\")\n#                 break\n            \n        total_loss = 0\n\n#clean_temporary_directories(strategy, manager_dir)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:49:09.128989Z","iopub.execute_input":"2024-08-14T18:49:09.129319Z","iopub.status.idle":"2024-08-14T19:08:58.598135Z","shell.execute_reply.started":"2024-08-14T18:49:09.12929Z","shell.execute_reply":"2024-08-14T19:08:58.597199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DetectionFromImageModule(DetectionInferenceModule):\n    def __init__(self, detection_model):\n        \n        sig = [tf.TensorSpec(shape=[1, None, None, 3],\n                             dtype=tf.uint8,\n                             name='input_tensor')]\n\n        def call_func(input_tensor):\n            return self._run_inference_on_images(input_tensor)\n\n        self.__call__ = tf.function(call_func, input_signature=sig)\n\n        super(DetectionFromImageModule, self).__init__(detection_model)\n        \n    def _run_inference_on_images(self, image, **kwargs):\n        label_id_offset = 1\n        image = tf.cast(image, tf.float32)\n        image, shapes = self._model.preprocess(image)\n        prediction_dict = self._model.predict(image, shapes, **kwargs)\n        detections = self._model.postprocess(prediction_dict, shapes)\n        classes_field = fields.DetectionResultFields.detection_classes\n        classes = tf.cast(detections[classes_field], tf.float32)\n        detections[classes_field] = (classes + label_id_offset)\n\n        for key, val in detections.items():\n            detections[key] = tf.cast(val, tf.float32)\n\n        return detections","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-08-14T19:43:08.124801Z","iopub.execute_input":"2024-08-14T19:43:08.125166Z","iopub.status.idle":"2024-08-14T19:43:08.14455Z","shell.execute_reply.started":"2024-08-14T19:43:08.125133Z","shell.execute_reply":"2024-08-14T19:43:08.143564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $OUTPUT_MODEL_DIR","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:43:48.00994Z","iopub.execute_input":"2024-08-14T18:43:48.010298Z","iopub.status.idle":"2024-08-14T18:43:49.131483Z","shell.execute_reply.started":"2024-08-14T18:43:48.010267Z","shell.execute_reply":"2024-08-14T18:43:49.130615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/saved_model","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:43:50.508699Z","iopub.execute_input":"2024-08-14T18:43:50.509084Z","iopub.status.idle":"2024-08-14T18:43:51.628562Z","shell.execute_reply.started":"2024-08-14T18:43:50.509045Z","shell.execute_reply":"2024-08-14T18:43:51.627634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detection_model","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:43:58.764612Z","iopub.execute_input":"2024-08-14T18:43:58.765034Z","iopub.status.idle":"2024-08-14T18:43:58.771487Z","shell.execute_reply.started":"2024-08-14T18:43:58.764998Z","shell.execute_reply":"2024-08-14T18:43:58.770475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n\nraw_image_dataset = tf.data.TFRecordDataset(TEST_DATASET)\nparsed_image_dataset = raw_image_dataset.map(parse_image_sample)\niterator = iter(parsed_image_dataset)\n\nfor idx in range(7):\n    image_features = next(iterator)\n\n    print(image_features.keys())\n    print(labels.keys())\n    \n    if len(image_features['image/object/class/text'].numpy()) == 0:\n        print(\"YOK\")\n    else:\n        print(\"VAR\")\n        \n        \n    image_raw = image_features['image/encoded']\n    image = tf.image.decode_jpeg(image_raw)\n    \n    # bunun sıralaması değişecek sanırım\n    \n    gt_boxes = np.stack([\n        image_features['image/object/bbox/xmin'],\n        image_features['image/object/bbox/ymin'],\n        image_features['image/object/bbox/xmax'],\n        image_features['image/object/bbox/ymax'],\n    ], -1)\n    \n    \n    \n    plot_img_with_boxes(image.numpy(), \n                        np.array([]), \n                        np.array([]),\n                        np.array([]))\n    \n    \n    \n    \n    image_true = image.numpy()\n    viz_utils.draw_bounding_boxes_on_image_array(image_true, \n                                                 gt_boxes,\n                                                 color='#5bf368')\n    \n    plot_img_with_boxes(image_true, \n                        np.array([]), \n                        np.array([]),\n                        np.array([]))\n    \n      \n    \n    image_float = tf.cast(image, dtype=tf.float32)\n    out = detection_model(tf.expand_dims(image_float, 0))   \n    \n    print(out.keys())\n\n    classes = out['detection_classes'].numpy()[0].astype('int')\n    scores = out['detection_scores'].numpy()[0]\n\n    boxes = out['detection_boxes'].numpy()[0]\n    \n    print(\"org\", gt_boxes, \"pred\", boxes[0:1])\n    \n#     boxes = np.stack([\n#         boxes[:,1],\n#         boxes[:,0],\n#         boxes[:,3],\n#         boxes[:,2]        \n#     ], -1)\n\n    image_pred = image.numpy()\n    \n    print(image_pred.shape)\n\n    viz_utils.draw_bounding_boxes_on_image_array(image_pred, \n                                                 boxes[0:1],\n                                                 color='yellow')\n    \n    print(classes[0], boxes[0], scores[0])\n    \n    plot_img_with_boxes(image_pred, \n                        np.array([]), \n                        np.array([]),\n                        np.array([]))\n\n\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:18:06.419148Z","iopub.execute_input":"2024-08-14T20:18:06.419513Z","iopub.status.idle":"2024-08-14T20:18:23.123261Z","shell.execute_reply.started":"2024-08-14T20:18:06.419476Z","shell.execute_reply":"2024-08-14T20:18:23.122102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ckpt = tf.train.Checkpoint(model=detection_model)\nmanager = tf.train.CheckpointManager(ckpt, \n                                     MODEL_DIR,\n                                     max_to_keep=1)\n\nstatus = ckpt.restore(manager.latest_checkpoint).expect_partial()\n\ndetection_module = DetectionFromImageModule(detection_model)\nconcrete_function = detection_module.__call__.get_concrete_function()\n#status.assert_existing_objects_matched()\n\nexported_checkpoint_manager = tf.train.CheckpointManager(ckpt, \n                                                         OUTPUT_MODEL_DIR, \n                                                         max_to_keep=1)\n\nexported_checkpoint_manager.save(checkpoint_number=0)\ntf.saved_model.save(detection_module,\n                    OUTPUT_MODEL_DIR + '/saved_model',\n                    signatures=concrete_function)","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:19:19.973522Z","iopub.execute_input":"2024-08-14T20:19:19.973913Z","iopub.status.idle":"2024-08-14T20:22:39.616337Z","shell.execute_reply.started":"2024-08-14T20:19:19.973878Z","shell.execute_reply":"2024-08-14T20:22:39.615431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_MODEL_DIR + '/saved_model'","metadata":{"execution":{"iopub.status.busy":"2024-08-02T23:15:42.569204Z","iopub.execute_input":"2024-08-02T23:15:42.569568Z","iopub.status.idle":"2024-08-02T23:15:42.598599Z","shell.execute_reply.started":"2024-08-02T23:15:42.56953Z","shell.execute_reply":"2024-08-02T23:15:42.597638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/saved_model","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:26:15.616016Z","iopub.execute_input":"2024-08-14T20:26:15.616369Z","iopub.status.idle":"2024-08-14T20:26:16.875835Z","shell.execute_reply.started":"2024-08-14T20:26:15.616337Z","shell.execute_reply":"2024-08-14T20:26:16.875011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r /kaggle/working/save_model.zip /kaggle/working/saved_model","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:26:26.115014Z","iopub.execute_input":"2024-08-14T20:26:26.115399Z","iopub.status.idle":"2024-08-14T20:26:36.800215Z","shell.execute_reply.started":"2024-08-14T20:26:26.115359Z","shell.execute_reply":"2024-08-14T20:26:36.799158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:27:02.329395Z","iopub.execute_input":"2024-08-14T20:27:02.329766Z","iopub.status.idle":"2024-08-14T20:27:03.565793Z","shell.execute_reply.started":"2024-08-14T20:27:02.329721Z","shell.execute_reply":"2024-08-14T20:27:03.564348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detector = tf.saved_model.load('/kaggle/working/saved_model/saved_model/')","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:27:17.068273Z","iopub.execute_input":"2024-08-14T20:27:17.068642Z","iopub.status.idle":"2024-08-14T20:28:21.737942Z","shell.execute_reply.started":"2024-08-14T20:27:17.068609Z","shell.execute_reply":"2024-08-14T20:28:21.737099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n\nraw_image_dataset = tf.data.TFRecordDataset(TEST_DATASET)\nparsed_image_dataset = raw_image_dataset.map(parse_image_sample)\niterator = iter(parsed_image_dataset)\n\nfor idx in range(7):\n    image_features = next(iterator)\n\n    print(image_features.keys())\n    print(labels.keys())\n    \n    if len(image_features['image/object/class/text'].numpy()) == 0:\n        print(\"YOK\")\n    else:\n        print(\"VAR\")\n        \n        \n    image_raw = image_features['image/encoded']\n    image = tf.image.decode_jpeg(image_raw)\n    \n    # bunun sıralaması değişecek sanırım\n    \n    gt_boxes = np.stack([\n        image_features['image/object/bbox/ymin'],\n        image_features['image/object/bbox/xmin'],\n        image_features['image/object/bbox/ymax'],\n        image_features['image/object/bbox/xmax'],\n    ], -1)\n    \n    \n    \n    plot_img_with_boxes(image.numpy(), \n                        np.array([]), \n                        np.array([]),\n                        np.array([]))\n    \n    \n    \n    \n    image_true = image.numpy()\n    viz_utils.draw_bounding_boxes_on_image_array(image_true, \n                                                 gt_boxes,\n                                                 color='#5bf368')\n    \n    plot_img_with_boxes(image_true, \n                        np.array([]), \n                        np.array([]),\n                        np.array([]))\n    \n    out = detector(tf.expand_dims(image, 0))\n    \n    print(out.keys())\n\n    classes = out['detection_classes'].numpy()[0].astype('int')\n    scores = out['detection_scores'].numpy()[0]\n\n    boxes = out['detection_boxes'].numpy()[0]\n    \n    print(\"org\", gt_boxes, \"pred\", boxes[0:1])\n    \n\n    image_pred = image.numpy()\n    \n    print(image_pred.shape)\n\n    viz_utils.draw_bounding_boxes_on_image_array(image_pred, \n                                                 boxes[0:1],\n                                                 color='yellow')\n    \n    print(classes[0], boxes[0], scores[0])\n    \n    plot_img_with_boxes(image_pred, \n                        np.array([]), \n                        np.array([]),\n                        np.array([]))","metadata":{"execution":{"iopub.status.busy":"2024-08-14T20:33:29.400085Z","iopub.execute_input":"2024-08-14T20:33:29.400491Z","iopub.status.idle":"2024-08-14T20:33:40.530028Z","shell.execute_reply.started":"2024-08-14T20:33:29.400446Z","shell.execute_reply":"2024-08-14T20:33:40.529179Z"},"trusted":true},"execution_count":null,"outputs":[]}]}