{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":52950,"databundleVersionId":5973250,"sourceType":"competition"},{"sourceId":4364795,"sourceType":"datasetVersion","datasetId":2274665}],"dockerImageVersionId":30198,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://github.com/ai-forever/hagrid/blob/master/images/demo.gif?raw=true)","metadata":{}},{"cell_type":"markdown","source":"### Dataset Download","metadata":{"execution":{"iopub.status.busy":"2022-06-20T10:33:56.943198Z","iopub.execute_input":"2022-06-20T10:33:56.943768Z","iopub.status.idle":"2022-06-20T10:34:46.363659Z","shell.execute_reply.started":"2022-06-20T10:33:56.943692Z","shell.execute_reply":"2022-06-20T10:34:46.362307Z"}}},{"cell_type":"code","source":"import urllib\n\nurllib.request.urlretrieve('https://sc.link/AO5l', 'subsample.zip')\n!mkdir -p dataset/\n!unzip -q subsample.zip -d dataset/subsample\n!rm -r subsample.zip\n!scp -r /kaggle/input/hagrid/ann_subsample /kaggle/working/dataset/ann_subsample","metadata":{"execution":{"iopub.status.busy":"2023-05-19T16:42:34.781184Z","iopub.execute_input":"2023-05-19T16:42:34.781601Z","iopub.status.idle":"2023-05-19T16:42:38.762681Z","shell.execute_reply.started":"2023-05-19T16:42:34.781564Z","shell.execute_reply":"2023-05-19T16:42:38.761416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport logging\nimport random\nfrom tqdm import tqdm\nfrom collections import defaultdict\nfrom typing import Tuple\nfrom glob import glob\n\nimport pandas as pd\nimport numpy as np\n\nfrom PIL import Image, ImageOps\nimport os\nfrom ipywidgets import interact\nfrom IPython.display import Image as DImage\nimport cv2\n\nimport torch\nfrom torch import nn, Tensor\nfrom torchvision import models\nfrom torchvision.transforms import Compose\nfrom torchvision.transforms import functional as F\nfrom torchvision import transforms as T\nfrom torchmetrics.detection.mean_ap import MeanAveragePrecision\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-05-19T16:41:07.113322Z","iopub.execute_input":"2023-05-19T16:41:07.11371Z","iopub.status.idle":"2023-05-19T16:41:14.86117Z","shell.execute_reply.started":"2023-05-19T16:41:07.113678Z","shell.execute_reply":"2023-05-19T16:41:14.860318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names = [\n   'call',\n   'dislike',\n   'fist',\n   'four',\n   'like',\n   'mute',\n   'ok',\n   'one',\n   'palm',\n   'peace_inverted',\n   'peace',\n   'rock',\n   'stop_inverted',\n   'stop',\n   'three',\n   'three2',\n   'two_up',\n   'two_up_inverted',\n   'no_gesture']\n\nFORMATS = (\".jpeg\", \".jpg\", \".jp2\", \".png\", \".tiff\", \".jfif\", \".bmp\", \".webp\", \".heic\")","metadata":{"execution":{"iopub.status.busy":"2023-05-19T16:41:14.862837Z","iopub.execute_input":"2023-05-19T16:41:14.863417Z","iopub.status.idle":"2023-05-19T16:41:14.86907Z","shell.execute_reply.started":"2023-05-19T16:41:14.863386Z","shell.execute_reply":"2023-05-19T16:41:14.868343Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset loader class","metadata":{}},{"cell_type":"code","source":"transform = T.ToTensor()\n\n\nclass GestureDataset(torch.utils.data.Dataset):\n\n    @staticmethod\n    def __get_files_from_dir(pth: str, extns: Tuple):\n        if not os.path.exists(pth):\n            print(f\"Dataset directory doesn't exist {pth}\")\n            return []\n        files = [f for f in os.listdir(pth) if f.endswith(extns)]\n        return files\n\n    def __read_annotations(self, path):\n        annotations_all = None\n        exists_images = []\n        for target in class_names:\n            path_to_csv = os.path.join(path, f\"{target}.json\")\n            if os.path.exists(path_to_csv):\n                json_annotation = json.load(open(\n                    os.path.join(path, f\"{target}.json\")\n                ))\n\n                json_annotation = [dict(annotation, **{\"name\": f\"{name}.jpg\"}) for name, annotation in\n                                   zip(json_annotation, json_annotation.values())]\n\n                annotation = pd.DataFrame(json_annotation)\n\n                annotation[\"target\"] = target\n                annotations_all = pd.concat([annotations_all, annotation], ignore_index=True)\n                exists_images.extend(\n                    self.__get_files_from_dir(os.path.join(self.path_images, target), FORMATS))\n            else:\n                if target != 'no_gesture':\n                    print(f\"Database for {target} not found\")\n\n        annotations_all[\"exists\"] = annotations_all[\"name\"].isin(exists_images)\n\n        annotations_all = annotations_all[annotations_all[\"exists\"]]\n\n        users = annotations_all[\"user_id\"].unique()\n        users = sorted(users)\n        random.Random(42).shuffle(users)\n        train_users = users[:int(len(users) * 0.8)]\n        val_users = users[int(len(users) * 0.8):]\n\n        annotations_all = annotations_all.copy()\n\n        if self.is_train:\n            annotations_all = annotations_all[annotations_all[\"user_id\"].isin(train_users)]\n        else:\n            annotations_all = annotations_all[annotations_all[\"user_id\"].isin(val_users)]\n\n        return annotations_all\n\n    def __init__(self, path_annotation, path_images, is_train, transform=None):\n        self.is_train = is_train\n        self.transform = transform\n        self.path_annotation = path_annotation\n        self.path_images = path_images\n        self.transform = transform\n        self.labels = {label: num for (label, num) in\n                       zip(class_names, range(len(class_names)))}\n        self.annotations = self.__read_annotations(self.path_annotation)\n\n    def __len__(self):\n        return self.annotations.shape[0]\n\n    def get_sample(self, index: int):\n        row = self.annotations.iloc[[index]].to_dict('records')[0]\n        image_pth = os.path.join(self.path_images, row[\"target\"], row[\"name\"])\n        image = Image.open(image_pth).convert(\"RGB\")\n\n        labels = torch.LongTensor([self.labels[label] for label in row[\"labels\"]])\n\n        target = {}\n        width, height = image.size\n\n        bboxes = []\n\n        for bbox in row[\"bboxes\"]:\n            x1, y1, w, h = bbox\n            bbox_abs = [x1 * width, y1 * height, (x1 + w) * width, (y1 + h) * height]\n            bboxes.append(bbox_abs)\n\n        target[\"labels\"] = labels\n        target[\"boxes\"] = torch.as_tensor(bboxes, dtype=torch.float32)\n        target[\"orig_size\"] = torch.as_tensor([int(height), int(width)])\n\n        return image, target\n\n    def __getitem__(self, index: int):\n        image, target = self.get_sample(index)\n        if self.transform:\n            image = self.transform(image)\n        return image, target","metadata":{"execution":{"iopub.status.busy":"2023-05-19T16:41:36.656071Z","iopub.execute_input":"2023-05-19T16:41:36.656767Z","iopub.status.idle":"2023-05-19T16:41:36.690539Z","shell.execute_reply.started":"2023-05-19T16:41:36.656721Z","shell.execute_reply":"2023-05-19T16:41:36.689535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setting some constants for training","metadata":{}},{"cell_type":"code","source":"random_seed = 42\nnum_classes = len(class_names)\nbatch_size = 16\nnum_epoch = 15\ntorch.manual_seed(random_seed)\nnp.random.seed(random_seed)\nrandom.seed(random_seed)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-05-24T17:51:33.925728Z","iopub.execute_input":"2023-05-24T17:51:33.926181Z","iopub.status.idle":"2023-05-24T17:51:34.021454Z","shell.execute_reply.started":"2023-05-24T17:51:33.926099Z","shell.execute_reply":"2023-05-24T17:51:34.019684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = GestureDataset(path_images='/kaggle/working/dataset/subsample',\n                            path_annotation='/kaggle/working/dataset/ann_subsample',\n                            is_train=True, transform=transform)\n\ntest_data = GestureDataset(path_images='/kaggle/working/dataset/subsample',\n                            path_annotation='/kaggle/working/dataset/ann_subsample',\n                            is_train=False, transform=transform)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-19T16:01:04.260912Z","iopub.status.idle":"2023-05-19T16:01:04.261991Z","shell.execute_reply.started":"2023-05-19T16:01:04.261735Z","shell.execute_reply":"2023-05-19T16:01:04.261761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    batch_targets = list()\n    images = list()\n\n    for b in batch:\n        images.append(b[0])\n        batch_targets.append({\"boxes\": b[1][\"boxes\"],\n                              \"labels\": b[1][\"labels\"]})\n    return images, batch_targets","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:11:57.841103Z","iopub.execute_input":"2023-05-16T22:11:57.841832Z","iopub.status.idle":"2023-05-16T22:11:57.847977Z","shell.execute_reply.started":"2023-05-16T22:11:57.84179Z","shell.execute_reply":"2023-05-16T22:11:57.84719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,collate_fn=collate_fn, shuffle=True, num_workers=4)\ntest_dataloader = torch.utils.data.DataLoader(test_data, batch_size=batch_size,collate_fn=collate_fn, shuffle=True, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:11:57.849171Z","iopub.execute_input":"2023-05-16T22:11:57.849624Z","iopub.status.idle":"2023-05-16T22:11:57.857976Z","shell.execute_reply.started":"2023-05-16T22:11:57.849581Z","shell.execute_reply":"2023-05-16T22:11:57.857105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading a pretrained framework and creating a model class","metadata":{}},{"cell_type":"code","source":"lr = 0.005\nmomentum = 0.9\nweight_decay = 5e-4","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:11:57.859434Z","iopub.execute_input":"2023-05-16T22:11:57.859813Z","iopub.status.idle":"2023-05-16T22:11:57.868282Z","shell.execute_reply.started":"2023-05-16T22:11:57.859777Z","shell.execute_reply":"2023-05-16T22:11:57.867497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = models.detection.ssdlite320_mobilenet_v3_large(num_classes=len(class_names) + 1, pretrained_backbone=True)\nmodel.to(device)\n\noptimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:11:57.870592Z","iopub.execute_input":"2023-05-16T22:11:57.871006Z","iopub.status.idle":"2023-05-16T22:12:03.902522Z","shell.execute_reply.started":"2023-05-16T22:11:57.87097Z","shell.execute_reply":"2023-05-16T22:12:03.901633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warmup_factor = 1.0 / 1000\nwarmup_iters = min(1000, len(train_data) - 1)\n\nlr_scheduler_warmup = torch.optim.lr_scheduler.LinearLR(\n    optimizer, start_factor=warmup_factor, total_iters=warmup_iters\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:12:03.903757Z","iopub.execute_input":"2023-05-16T22:12:03.904631Z","iopub.status.idle":"2023-05-16T22:12:03.910025Z","shell.execute_reply.started":"2023-05-16T22:12:03.904583Z","shell.execute_reply":"2023-05-16T22:12:03.909122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation function for calculate mAP metric","metadata":{}},{"cell_type":"code","source":"def eval(model, test_dataloader, epoch):\n    model.eval()\n    with torch.no_grad():\n        mapmetric = MeanAveragePrecision()\n        \n        for images, targets in test_dataloader:\n            images = list(image.to(device) for image in images)\n            output = model(images)\n            \n            for pred in output:\n                for key, value in pred.items():\n                    pred[key] = value.cpu()\n                    \n            mapmetric.update(output, targets)\n\n    metrics = mapmetric.compute()\n    return metrics","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:12:27.654253Z","iopub.execute_input":"2023-05-16T22:12:27.654651Z","iopub.status.idle":"2023-05-16T22:12:27.662422Z","shell.execute_reply.started":"2023-05-16T22:12:27.654618Z","shell.execute_reply":"2023-05-16T22:12:27.661513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training loop","metadata":{}},{"cell_type":"code","source":"!mkdir checkpoints\nfor epoch in range(num_epoch):\n    model.train()\n    total = 0\n    sum_loss = 0\n    for images, targets in tqdm(train_dataloader):\n        batch = len(images)\n        images = list(image.to(device) for image in images)\n        for target in targets:\n            for key, value in target.items():\n                target[key] = value.to(device)\n        loss_dict = model(images, targets)\n        losses = sum(loss for loss in loss_dict.values())\n        loss = losses.item()\n\n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n\n        lr_scheduler_warmup.step()\n\n        total = total + batch\n        sum_loss = sum_loss + loss\n    metrics = eval(model, test_dataloader, epoch)\n    print(f\"epoch : {epoch}  |||  loss : {sum_loss / total} ||| MAP : {metrics['map']}\")\ntorch.save(model.state_dict(),f\"checkpoints/{epoch}.pth\")","metadata":{"execution":{"iopub.status.busy":"2023-05-16T22:12:32.366151Z","iopub.execute_input":"2023-05-16T22:12:32.366561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test model","metadata":{}},{"cell_type":"code","source":"images = []\nfor gesture in class_names[:-1]:\n    image_path = glob(f'/kaggle/working/dataset/subsample/{gesture}/*.jpg')[0]\n    images.append(Image.open(image_path))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_tensors = images.copy()\nimages_tensors_input = list(transform(image).to(device) for image in images_tensors)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    model.eval()\n    out = model(images_tensors_input)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Set the score **threshold = 0.2** because the training model on a small dataset will be of poor quality","metadata":{}},{"cell_type":"code","source":"bboxes = []\nscores = []\nlabels = []\nfor pred in out:\n    ids = pred['scores'] >= 0.2\n    bboxes.append(pred['boxes'][ids][:2].cpu().numpy().astype(np.int))\n    scores.append(pred['scores'][ids][:2].cpu().numpy())\n    labels.append(pred['labels'][ids][:2].cpu().numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_class_names = []\n\nfor name in class_names:\n    if name == 'stop_inverted':\n        short_class_names.append('stop inv.')\n    elif name == 'peace_inverted':\n        short_class_names.append('peace inv.')\n    elif name == 'two_up':\n        short_class_names.append('two up')\n    elif name == 'two_up_inverted':\n        short_class_names.append('two up inv.')\n    elif name == 'no_gesture':\n        short_class_names.append('no gesture')\n    else:\n        short_class_names.append(name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_images = []\nfor bbox, score, label, image in zip(bboxes, scores, labels, images):\n    image = np.array(image)\n    for i, box in enumerate(bbox):\n        _,width,_  = image.shape\n        image = cv2.rectangle(image, box[:2], box[2:], thickness=3, color=[255, 0, 255])\n        cv2.putText(image, f'{short_class_names[label[i]]}: {score[i]:0.2f}', (box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX,\n                        width / 780, (0, 0, 255), 2)\n    final_images.append(Image.fromarray(image))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir out_images\nout_images = []\nfor i, image in enumerate(final_images):\n    out_name = f\"out_images/{i}.png\"\n    out_images.append(out_name)\n    image.save(out_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now we can look at the results of the model.","metadata":{}},{"cell_type":"code","source":"out_dir = \"out_images/\"\n@interact\ndef show_images(file=os.listdir(out_dir)):\n    display(DImage(out_dir+file, width=600, height=300))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## As we can see there are some problems with gesture detection. \n### This is due to the fact that the model was trained on a small data set. To get a better model train it on a full [Hagrid dataset](https://github.com/hukenovs/hagrid).","metadata":{}},{"cell_type":"markdown","source":"### Thanks to https://www.kaggle.com/code/karinakvanchiani/example-of-data-usage","metadata":{}},{"cell_type":"markdown","source":"![](https://github.com/hukenovs/hagrid/blob/master/images/hagrid.jpg?raw=true)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}