Inspired by here I've tried implementing the Probabilistic F Score in Tensorflow, Torch, and Numpy. Instead of using for-loop, we can save some time with matrix operation. Check out the notebook for speed comparison.
This will give us overall pFBeta score during rather than batch-wise score during training. This perk comes from using tf.kreas.metrics.Metric.
(tf.keras.metrics.Metric):\n \n ():\n ().__init__(name=name, **kwargs)\n self.beta = beta\n self.epsilon = epsilon\n self.pos = self.add_weight(name=, initializer=)\n self.ctp = self.add_weight(name=, initializer=)\n self.cfp = self.add_weight(name=, initializer=)\n\n ():\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.clip_by_value(y_pred, , )\n pos = tf.reduce_sum(y_true)\n ctp = tf.reduce_sum(y_pred[y_true==])\n cfp = tf.reduce_sum(y_pred[y_true==])\n self.pos.assign_add(pos)\n self.ctp.assign_add(ctp)\n self.cfp.assign_add(cfp)\n\n ():\n beta_squared = self.beta * self.beta\n c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n c_recall = self.ctp / (self.pos + self.epsilon)\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n tf.cond(c_precision > c_recall > , : result, : )\n ():\n preds = tf.clip_by_value(preds, , )\n y_true_count = tf.reduce_sum(labels)\n ctp = tf.reduce_sum(preds[labels==])\n cfp = tf.reduce_sum(preds[labels==])\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n (c_precision > c_recall > ):\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n result\n :\n \n ():\n preds = preds.clip(, )\n y_true_count = labels.()\n ctp = preds[labels==].()\n cfp = preds[labels==].()\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n (c_precision > c_recall > ):\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n result\n :\n \nInspired by here I've tried implementing the Probabilistic F Score in Tensorflow, Torch, and Numpy. Instead of using for-loop, we can save some time with matrix operation. Check out the notebook for speed comparison.
This will give us overall pFBeta score during rather than batch-wise score during training. This perk comes from using tf.kreas.metrics.Metric.
(tf.keras.metrics.Metric):\n \n ():\n ().__init__(name=name, **kwargs)\n self.beta = beta\n self.epsilon = epsilon\n self.pos = self.add_weight(name=, initializer=)\n self.ctp = self.add_weight(name=, initializer=)\n self.cfp = self.add_weight(name=, initializer=)\n\n ():\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.clip_by_value(y_pred, , )\n pos = tf.reduce_sum(y_true)\n ctp = tf.reduce_sum(y_pred[y_true==])\n cfp = tf.reduce_sum(y_pred[y_true==])\n self.pos.assign_add(pos)\n self.ctp.assign_add(ctp)\n self.cfp.assign_add(cfp)\n\n ():\n beta_squared = self.beta * self.beta\n c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n c_recall = self.ctp / (self.pos + self.epsilon)\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n tf.cond(c_precision > c_recall > , : result, : )\n ():\n preds = tf.clip_by_value(preds, , )\n y_true_count = tf.reduce_sum(labels)\n ctp = tf.reduce_sum(preds[labels==])\n cfp = tf.reduce_sum(preds[labels==])\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n (c_precision > c_recall > ):\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n result\n :\n \n ():\n preds = preds.clip(, )\n y_true_count = labels.()\n ctp = preds[labels==].()\n cfp = preds[labels==].()\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n (c_precision > c_recall > ):\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n result\n :\n \nIs beta==1. for this competition?
yes, F1 score means Beta=1
", "rawMarkdown": "yes, F1 score means Beta=1", "votes": 2 } ] }, { "id": 2079733, "postDate": "2022-12-29T15:05:13.740Z", "content": "Just realized we can get overall pFBeta score for tensorflow during training using tf.keras.metrics.Metric, here's the following code. It holds intermediate state hence can produce overall result unlike simply function metric.
(tf.keras.metrics.Metric):\n \n ():\n ().__init__(name=name, **kwargs)\n self.beta = beta\n self.epsilon = epsilon\n self.pos = self.add_weight(name=, initializer=)\n self.ctp = self.add_weight(name=, initializer=)\n self.cfp = self.add_weight(name=, initializer=)\n\n ():\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.clip_by_value(y_pred, , )\n pos = tf.reduce_sum(y_true)\n ctp = tf.reduce_sum(y_pred[y_true==])\n cfp = tf.reduce_sum(y_pred[y_true==])\n self.pos.assign_add(pos)\n self.ctp.assign_add(ctp)\n self.cfp.assign_add(cfp)\n\n ():\n beta_squared = self.beta * self.beta\n c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n c_recall = self.ctp / (self.pos + self.epsilon)\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n tf.cond(c_precision > c_recall > , : result, : )\nhi, I noticed that the code Probabilistic F Score modifies the calculation of cfp and cfp += 1 - prediction was deleted.
\nThus I modified the calculation code of pytorch/numpy as follows:
():\n preds = preds.clip(, )\n y_true_count = labels.()\n ctp = preds[labels==].()\n \n cfp = preds[labels==].()\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp + )\n c_recall = ctp / (y_true_count + )\n (c_precision > c_recall > ):\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + )\n result\n :\n \nThanks for informing me
", "rawMarkdown": "Thanks for informing me" } ] }, { "id": 2079631, "postDate": "2022-12-29T14:04:59.360Z", "content": "@awsaf49 Thanks for sharing
", "rawMarkdown": "@awsaf49 Thanks for sharing", "votes": 2 } ], "comments": [ { "id": 2052610, "author_name": "Vladimir Slaykovskiy", "author_url": "", "post_date": "2022-12-02T10:43:54.017000", "content": "Is beta==1. for this competition?
yes, F1 score means Beta=1
", "votes": 2, "replies": [] } ] }, { "id": 2079733, "author_name": "Awsaf", "author_url": "", "post_date": "2022-12-29T15:05:13.740000", "content": "Just realized we can get overall pFBeta score for tensorflow during training using tf.keras.metrics.Metric, here's the following code. It holds intermediate state hence can produce overall result unlike simply function metric.
(tf.keras.metrics.Metric):\n \n ():\n ().__init__(name=name, **kwargs)\n self.beta = beta\n self.epsilon = epsilon\n self.pos = self.add_weight(name=, initializer=)\n self.ctp = self.add_weight(name=, initializer=)\n self.cfp = self.add_weight(name=, initializer=)\n\n ():\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.clip_by_value(y_pred, , )\n pos = tf.reduce_sum(y_true)\n ctp = tf.reduce_sum(y_pred[y_true==])\n cfp = tf.reduce_sum(y_pred[y_true==])\n self.pos.assign_add(pos)\n self.ctp.assign_add(ctp)\n self.cfp.assign_add(cfp)\n\n ():\n beta_squared = self.beta * self.beta\n c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n c_recall = self.ctp / (self.pos + self.epsilon)\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n tf.cond(c_precision > c_recall > , : result, : )\nhi, I noticed that the code Probabilistic F Score modifies the calculation of cfp and cfp += 1 - prediction was deleted.
\nThus I modified the calculation code of pytorch/numpy as follows:
():\n preds = preds.clip(, )\n y_true_count = labels.()\n ctp = preds[labels==].()\n \n cfp = preds[labels==].()\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp + )\n c_recall = ctp / (y_true_count + )\n (c_precision > c_recall > ):\n result = ( + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + )\n result\n :\n \nThanks for informing me
", "votes": 0, "replies": [] } ] }, { "id": 2079631, "author_name": "Shailesh Kumar", "author_url": "", "post_date": "2022-12-29T14:04:59.360000", "content": "@awsaf49 Thanks for sharing
", "votes": 2, "replies": [] } ], "raw_markdown_by_id": { "2048535": "Inspired by [here](https://www.kaggle.com/code/sohier/probabilistic-f-score) I've tried implementing the `Probabilistic F Score` in Tensorflow, Torch, and Numpy. Instead of using for-loop, we can save some time with matrix operation. Check out the notebook for speed comparison.\n\n* Notebook: [Metric: Probabilistic FScore [TF, Torch, Numpy]](https://www.kaggle.com/awsaf49/metric-probabilistic-fscore-tf-torch-numpy/)\n\n## TensorFlow:\n\n### Overall\nThis will give us overall pFBeta score during rather than batch-wise score during training. This perk comes from using `tf.kreas.metrics.Metric`. \n\n```py\nclass pFBeta(tf.keras.metrics.Metric):\n \"\"\"Compute overall probabilistic F-beta score.\"\"\"\n def __init__(self, beta=1, epsilon=1e-5, name='pfbeta', **kwargs):\n super().__init__(name=name, **kwargs)\n self.beta = beta\n self.epsilon = epsilon\n self.pos = self.add_weight(name='pos', initializer='zeros')\n self.ctp = self.add_weight(name='ctp', initializer='zeros')\n self.cfp = self.add_weight(name='cfp', initializer='zeros')\n \n def update_state(self, y_true, y_pred, sample_weight=None):\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.clip_by_value(y_pred, 0, 1)\n pos = tf.reduce_sum(y_true)\n ctp = tf.reduce_sum(y_pred[y_true==1])\n cfp = tf.reduce_sum(y_pred[y_true==0])\n self.pos.assign_add(pos)\n self.ctp.assign_add(ctp)\n self.cfp.assign_add(cfp)\n \n def result(self):\n beta_squared = self.beta * self.beta\n c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n c_recall = self.ctp / (self.pos + self.epsilon)\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return tf.cond(c_precision > 0 and c_recall > 0, lambda: result, lambda: 0.0)\n```\n\n### Batchwise\n```py\ndef pfbeta_tf(labels, preds, beta=1):\n preds = tf.clip_by_value(preds, 0, 1)\n y_true_count = tf.reduce_sum(labels)\n ctp = tf.reduce_sum(preds[labels==1])\n cfp = tf.reduce_sum(preds[labels==0])\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n if (c_precision > 0 and c_recall > 0):\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return result\n else:\n return 0.0\n```\n\n### PyTorch/Numpy\n```py\ndef pfbeta_torch(labels, preds, beta=1):\n preds = preds.clip(0, 1)\n y_true_count = labels.sum()\n ctp = preds[labels==1].sum()\n cfp = preds[labels==0].sum()\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n if (c_precision > 0 and c_recall > 0):\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return result\n else:\n return 0.0\n```", "2052610": "Is `beta==1.` for this competition?", "2079733": "Just realized we can get overall pFBeta score for tensorflow during training using `tf.keras.metrics.Metric`, here's the following code. It holds intermediate state hence can produce overall result unlike simply function metric.\n\n```py\nclass pFBeta(tf.keras.metrics.Metric):\n \"\"\"Compute overall probabilistic F-beta score.\"\"\"\n def __init__(self, beta=1, epsilon=1e-5, name='pfbeta', **kwargs):\n super().__init__(name=name, **kwargs)\n self.beta = beta\n self.epsilon = epsilon\n self.pos = self.add_weight(name='pos', initializer='zeros')\n self.ctp = self.add_weight(name='ctp', initializer='zeros')\n self.cfp = self.add_weight(name='cfp', initializer='zeros')\n \n def update_state(self, y_true, y_pred, sample_weight=None):\n y_true = tf.cast(y_true, tf.float32)\n y_pred = tf.clip_by_value(y_pred, 0, 1)\n pos = tf.reduce_sum(y_true)\n ctp = tf.reduce_sum(y_pred[y_true==1])\n cfp = tf.reduce_sum(y_pred[y_true==0])\n self.pos.assign_add(pos)\n self.ctp.assign_add(ctp)\n self.cfp.assign_add(cfp)\n \n def result(self):\n beta_squared = self.beta * self.beta\n c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n c_recall = self.ctp / (self.pos + self.epsilon)\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return tf.cond(c_precision > 0 and c_recall > 0, lambda: result, lambda: 0.0)\n```", "2052347": "hi, I noticed that the code [Probabilistic F Score](https://www.kaggle.com/code/sohier/probabilistic-f-score/notebook) modifies the calculation of cfp and `cfp += 1 - prediction` was deleted.\nThus I modified the calculation code of pytorch/numpy as follows:\n``` python\ndef pfbeta_torch(labels, preds, beta=1):\n preds = preds.clip(0, 1)\n y_true_count = labels.sum()\n ctp = preds[labels==1].sum()\n # cfp = (1 - preds[labels==1]).sum() + (preds[labels==0]).sum()\n cfp = preds[labels==0].sum()\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp + 1e-4)\n c_recall = ctp / (y_true_count + 1e-4)\n if (c_precision > 0 and c_recall > 0):\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + 1e-4)\n return result\n else:\n return 0.0\n```\n", "2079631": "@awsaf49 Thanks for sharing" } }