{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":37333,"databundleVersionId":3949526,"sourceType":"competition"},{"sourceId":6774400,"sourceType":"datasetVersion","datasetId":3895136},{"sourceId":7869901,"sourceType":"datasetVersion","datasetId":4617753},{"sourceId":7869997,"sourceType":"datasetVersion","datasetId":4617825},{"sourceId":7870715,"sourceType":"datasetVersion","datasetId":4618317},{"sourceId":7871004,"sourceType":"datasetVersion","datasetId":4618499},{"sourceId":7871459,"sourceType":"datasetVersion","datasetId":4618776},{"sourceId":8108156,"sourceType":"datasetVersion","datasetId":4789280},{"sourceId":8241724,"sourceType":"datasetVersion","datasetId":4889051},{"sourceId":8245346,"sourceType":"datasetVersion","datasetId":4621378}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/pyradd/pyradiomics/*.whl\n!yes | dpkg -i --force-depends /kaggle/input/pyvips-python-and-deb-package/linux_packages/archives/*.deb\n!pip install pyvips -f /kaggle/input/pyvips-python-and-deb-package/python_packages/ --no-index\n!pip list | grep pyvips\n!cp -r /kaggle/input/yyyyyyyyyyyy/pyradiomics-3.0.1 /kaggle/working\n!pip install --no-index --find-links /kaggle/working/pyradiomics-3.0.1/ -r requirements.txt\n%cd /kaggle/working/pyradiomics-3.0.1\n!python setup.py build_ext --inplace\nimport pyvips\nfrom radiomics import featureextractor\nfrom radiomics import imageoperations","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:44:57.019404Z","iopub.execute_input":"2024-04-17T18:44:57.020027Z","iopub.status.idle":"2024-04-17T18:47:20.629437Z","shell.execute_reply.started":"2024-04-17T18:44:57.019987Z","shell.execute_reply":"2024-04-17T18:47:20.628355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport tensorflow.keras.layers as l\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nimport tifffile","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:47:20.630988Z","iopub.execute_input":"2024-04-17T18:47:20.63136Z","iopub.status.idle":"2024-04-17T18:47:32.367591Z","shell.execute_reply.started":"2024-04-17T18:47:20.631332Z","shell.execute_reply":"2024-04-17T18:47:32.366587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Disable the decompression bomb limit\nImage.MAX_IMAGE_PIXELS = None","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:47:32.368794Z","iopub.execute_input":"2024-04-17T18:47:32.369298Z","iopub.status.idle":"2024-04-17T18:47:32.373725Z","shell.execute_reply.started":"2024-04-17T18:47:32.369271Z","shell.execute_reply":"2024-04-17T18:47:32.372696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/mayo-clinic-strip-ai/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:47:32.376339Z","iopub.execute_input":"2024-04-17T18:47:32.376704Z","iopub.status.idle":"2024-04-17T18:47:32.42412Z","shell.execute_reply.started":"2024-04-17T18:47:32.376667Z","shell.execute_reply":"2024-04-17T18:47:32.42342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tooBig = []","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:47:32.425321Z","iopub.execute_input":"2024-04-17T18:47:32.425685Z","iopub.status.idle":"2024-04-17T18:47:32.429999Z","shell.execute_reply.started":"2024-04-17T18:47:32.425654Z","shell.execute_reply":"2024-04-17T18:47:32.429045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_files_by_size(folder_path):\n  \"\"\"\n  This function gets a list of all files in a folder sorted by size.\n\n  Args:\n      folder_path: The path to the folder to search.\n\n  Returns:\n      A list of tuples containing (file_path, file_size) sorted by size in descending order.\n  \"\"\"\n  files_with_size = []\n  for root, _, files in os.walk(folder_path):\n    for file in files:\n      file_path = os.path.join(root, file)\n      try:\n        file_size = os.path.getsize(file_path)\n        files_with_size.append((file_path, file_size))\n      except OSError:\n        # Handle potential errors like permission issues\n        pass\n  # Sort the list by size in descending order using lambda and sorted\n  files_with_size.sort(key=lambda x: x[1], reverse=True)\n  return files_with_size\n\n# Example usage\nfolder_path = \"/kaggle/input/mayo-clinic-strip-ai/test\"\nsorted_files = get_files_by_size(folder_path)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:47:32.431055Z","iopub.execute_input":"2024-04-17T18:47:32.431307Z","iopub.status.idle":"2024-04-17T18:47:32.450515Z","shell.execute_reply.started":"2024-04-17T18:47:32.431285Z","shell.execute_reply":"2024-04-17T18:47:32.449667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_files = np.array(sorted_files)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:47:32.451703Z","iopub.execute_input":"2024-04-17T18:47:32.451998Z","iopub.status.idle":"2024-04-17T18:47:32.456524Z","shell.execute_reply.started":"2024-04-17T18:47:32.451975Z","shell.execute_reply":"2024-04-17T18:47:32.45575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tile(img, sz=128, N=16):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N] # pick up Top N dark tiles\n    img = img[idxs]\n    return img\n\ndef save_dataset(\n    df: np.array,\n    N=16,\n    max_size=20000, \n    crop_size=1024, \n    image_dir='../input/mayo-clinic-strip-ai/train', \n    out_dir='./train_data',\n):\n    format_to_dtype = {\n       'uchar': np.uint8,\n       'char': np.int8,\n       'ushort': np.uint16,\n       'short': np.int16,\n       'uint': np.uint32,\n       'int': np.int32,\n       'float': np.float32,\n       'double': np.float64,\n       'complex': np.complex64,\n       'dpcomplex': np.complex128,\n    }\n    \n    if not os.path.isdir(out_dir):\n        os.makedirs(out_dir)\n        \n    tk0 = tqdm(enumerate(df), total=len(df))\n    for i, image_path in tk0:\n        image_id = '.'.join('/'.join(image_path.split('/')[-1:]).split('.')[:-1])\n        print(f\"[{i+1}/{len(df)}] image_id: {image_id}\")\n        image = pyvips.Image.thumbnail(image_path, max_size, height=max_size)\n        image = np.array(image)\n        width, height, c = image.shape\n        image = tile(image, sz=crop_size, N=N)\n        for idx, img in enumerate(image):\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n            cv2.imwrite(f\"{out_dir}/{image_id}_{idx}.jpg\", img, [cv2.IMWRITE_JPEG_QUALITY, 100])\n\n        del img, image; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:48:18.5654Z","iopub.execute_input":"2024-04-17T18:48:18.565838Z","iopub.status.idle":"2024-04-17T18:48:18.580917Z","shell.execute_reply.started":"2024-04-17T18:48:18.565806Z","shell.execute_reply":"2024-04-17T18:48:18.579901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dataset(\n    sorted_files[:,0],\n    N=16, \n    max_size=20000,\n    crop_size=1024, \n    image_dir=\"/kaggle/input/mayo-clinic-strip-ai/test\", \n    out_dir=f'/kaggle/working/test'\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T18:48:20.704081Z","iopub.execute_input":"2024-04-17T18:48:20.704708Z","iopub.status.idle":"2024-04-17T18:49:41.429763Z","shell.execute_reply.started":"2024-04-17T18:48:20.704674Z","shell.execute_reply":"2024-04-17T18:49:41.428819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport sys\nimport zipfile\nfrom IPython.display import FileLink\nfrom pathlib import Path\nimport albumentations as A\nimport SimpleITK as sitk\nimport csv\nfrom scipy.stats import mannwhitneyu\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\nimport matplotlib.cm as cm\nfrom sklearn.metrics import classification_report\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:27.879253Z","iopub.execute_input":"2024-04-14T16:08:27.8796Z","iopub.status.idle":"2024-04-14T16:08:28.998122Z","shell.execute_reply.started":"2024-04-14T16:08:27.879568Z","shell.execute_reply":"2024-04-14T16:08:28.997252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = pd.read_csv(\"/kaggle/input/pyradiomcs-features/pyradiomics_features.csv\")","metadata":{"id":"npZwq_30H-qo","execution":{"iopub.status.busy":"2024-04-14T16:08:28.999237Z","iopub.execute_input":"2024-04-14T16:08:28.999766Z","iopub.status.idle":"2024-04-14T16:08:31.414819Z","shell.execute_reply.started":"2024-04-14T16:08:28.999743Z","shell.execute_reply":"2024-04-14T16:08:31.41368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = train_data_df.sample(frac=1, random_state=69)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:31.416477Z","iopub.execute_input":"2024-04-14T16:08:31.416808Z","iopub.status.idle":"2024-04-14T16:08:31.458823Z","shell.execute_reply.started":"2024-04-14T16:08:31.416782Z","shell.execute_reply":"2024-04-14T16:08:31.457856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = train_data_df.replace(np.nan, 0)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:31.463765Z","iopub.execute_input":"2024-04-14T16:08:31.46411Z","iopub.status.idle":"2024-04-14T16:08:31.622934Z","shell.execute_reply.started":"2024-04-14T16:08:31.464082Z","shell.execute_reply":"2024-04-14T16:08:31.622047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = train_data_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:31.624219Z","iopub.execute_input":"2024-04-14T16:08:31.624877Z","iopub.status.idle":"2024-04-14T16:08:31.643151Z","shell.execute_reply.started":"2024-04-14T16:08:31.624841Z","shell.execute_reply":"2024-04-14T16:08:31.642161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:31.644402Z","iopub.execute_input":"2024-04-14T16:08:31.644689Z","iopub.status.idle":"2024-04-14T16:08:31.654677Z","shell.execute_reply.started":"2024-04-14T16:08:31.644665Z","shell.execute_reply":"2024-04-14T16:08:31.653545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _weighted_mc_log_loss(y_true, y_pred, epsilon=1e-15):\n    class_cnt = [sum(int(val == cl) for val in y_true) for cl in range(2)]\n    w = [0.5 for _ in range(2)]\n    return -sum(\n        w[cl] * sum(\n            (y == cl) / class_cnt[cl] * np.log(max(min(y_hat, 1 - epsilon), epsilon))\n            for y, y_hat in zip(y_true, y_pred[:, cl])\n        )\n        for cl in range(2)\n    ) / sum(w[cl] for cl in range(2))","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:31.655951Z","iopub.execute_input":"2024-04-14T16:08:31.656293Z","iopub.status.idle":"2024-04-14T16:08:31.664246Z","shell.execute_reply.started":"2024-04-14T16:08:31.656266Z","shell.execute_reply":"2024-04-14T16:08:31.663264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming train_data_df is your DataFrame containing the data\n# Replace 'Label' with the actual column name if it's different\n\n\n# Initial counts\nnum_CE = len(train_data_df[train_data_df['Label'] == 0])\nnum_LAA = len(train_data_df[train_data_df['Label'] == 1])\n\n# Calculate the difference in counts between the two classes\ncount_diff = num_CE - num_LAA\n\n# If there are more records with 'CE' label, randomly sample records with 'CE' to match the count of 'LAA'\nif count_diff > 0:\n    train_data_df = train_data_df.drop(train_data_df[train_data_df['Label'] == 0].sample(count_diff).index)\n# If there are more records with 'LAA' label, randomly sample records with 'LAA' to match the count of 'CE'\nelif count_diff < 0:\n    train_data_df = train_data_df.drop(train_data_df[train_data_df['Label'] == 1].sample(-count_diff).index)\n\n# Write the modified DataFrame to a new CSV file\ntrain_data_df.to_csv(\"modified_train_data.csv\", index=False)\n\nprint(\"Number of records with 'CE' and 'LAA' labels are now equal.\")\nprint(\"New size of train_data_df:\", len(train_data_df))\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:31.665411Z","iopub.execute_input":"2024-04-14T16:08:31.66572Z","iopub.status.idle":"2024-04-14T16:08:36.013811Z","shell.execute_reply.started":"2024-04-14T16:08:31.665696Z","shell.execute_reply":"2024-04-14T16:08:36.012838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"# Function to extract prefix\ndef extract_prefix(image_id):\n    return '_'.join(image_id.split('_')[:-1])\n\n\n# Initialize DataFrames to store the results\ntrain_big_df = pd.DataFrame()\ntest_big_df = pd.DataFrame()\n\n# Extract the 'Image_ID' column\nimage_id_column = train_data_df['Image_ID']\n\n# Remove non-numeric data from all columns except 'Image_ID'\nfor col in train_data_df.columns:\n    if col != 'Image_ID':\n        train_data_df[col] = pd.to_numeric(train_data_df[col], errors='coerce')\n\n# Add back the 'Image_ID' column\ntrain_data_df['Image_ID'] = image_id_column\n\n# Split the data into train and test based on the prefix of 'Image_ID'\ntrain_mask = train_data_df['Image_ID'].apply(lambda x: extract_prefix(x) in train_df['image_id'].values)\ntrain_big_chunk_df = train_data_df[train_mask]\ntest_big_chunk_df = train_data_df[~train_mask]\n\n# Concatenate DataFrames for the current chunk with the main DataFrames\ntrain_big_df = pd.concat([train_big_df, train_big_chunk_df], ignore_index=True)\ntest_big_df = pd.concat([test_big_df, test_big_chunk_df], ignore_index=True)\n\n# Optional: Drop duplicate rows from the resulting DataFrames\ntrain_big_df = train_big_df.drop_duplicates()\ntest_big_df = test_big_df.drop_duplicates()\n\n# Optional: Reset the index of the resulting DataFrames\ntrain_big_df.reset_index(drop=True, inplace=True)\ntest_big_df.reset_index(drop=True, inplace=True)\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:36.021518Z","iopub.execute_input":"2024-04-14T16:08:36.021812Z","iopub.status.idle":"2024-04-14T16:08:36.035472Z","shell.execute_reply.started":"2024-04-14T16:08:36.021787Z","shell.execute_reply":"2024-04-14T16:08:36.034498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Run this in case you choose some features only**","metadata":{"id":"68h1le_zC9CO"}},{"cell_type":"code","source":"# Assuming train_big_df is your DataFrame containing the data\ntrain_big_df = train_data_df\nfeatures = train_big_df.iloc[:, 2:]\n\n# Only consider numeric columns\nnumeric_cols = features.columns[features.apply(lambda x: pd.to_numeric(x, errors='coerce').notnull().all())]\nfeatures = features[numeric_cols]\n\n# Assuming 'Label' is the target column\ntarget_column = 'Label'\nclass_labels = train_big_df[target_column].unique()\n\nu_test_results = {}\n\nfor feature in features.columns:\n    feature_results = {}\n    for i in range(len(class_labels)):\n        for j in range(i+1, len(class_labels)):\n            group1 = train_big_df[train_big_df[target_column] == class_labels[i]][feature]\n            group2 = train_big_df[train_big_df[target_column] == class_labels[j]][feature]\n            stat, _ = mannwhitneyu(group1, group2)  # We don't need p-value here\n            pair_label = f\"{class_labels[i]} vs {class_labels[j]}\"\n            feature_results[pair_label] = {'statistic': stat}\n    u_test_results[feature] = feature_results\n\nu_test_results_df = pd.DataFrame(u_test_results)\n\n# Selecting features directly without considering P-values\nselected_features_list = u_test_results_df.columns.tolist()\n\n# Add the target column to the selected features list\nselected_features_list = ['Label'] + selected_features_list\n\n# Filter the DataFrame with selected features\ntrain_big_df = train_big_df[selected_features_list]\n#test_big_df = test_big_df[selected_features_list]","metadata":{"id":"mi17YpNi5-va","outputId":"95938b2e-57a1-4993-ce80-03ef44416c21","execution":{"iopub.status.busy":"2024-04-14T16:08:36.036748Z","iopub.execute_input":"2024-04-14T16:08:36.037118Z","iopub.status.idle":"2024-04-14T16:08:42.493681Z","shell.execute_reply.started":"2024-04-14T16:08:36.037086Z","shell.execute_reply":"2024-04-14T16:08:42.49255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_p_value(feature, target, data):\n    group1 = data[data[target] == 0][feature]  # Group 1: label 0\n    group2 = data[data[target] == 1][feature]  # Group 2: label 1\n    stat, p_value = mannwhitneyu(group1, group2)\n    return p_value\n\n# Target variable\ntarget_variable = 'Label'\n\n# Threshold for significance\nthreshold = 0.07\n\n# Dictionary to store p-values for each feature\np_values = {}\n\n# Iterate over each feature in the DataFrame\nfor idx, feature in enumerate(train_big_df.columns):\n    if idx >= 2:  # Skip the first two columns\n        if feature != target_variable:\n            # Calculate p-value for the current feature\n            p_value = calculate_p_value(feature, target_variable, train_big_df)\n            p_values[feature] = p_value\n\n# Filter significant features based on the threshold\nsignificant_features = [feature for feature, p_value in p_values.items() if p_value <= threshold]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:42.495094Z","iopub.execute_input":"2024-04-14T16:08:42.495467Z","iopub.status.idle":"2024-04-14T16:08:46.074795Z","shell.execute_reply.started":"2024-04-14T16:08:42.495431Z","shell.execute_reply":"2024-04-14T16:08:46.073821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_red = train_big_df.drop(columns=[col for col in train_big_df.columns if col not in significant_features and col not in [\"Image_ID\", \"Label\"]])\n\ntrain_red","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:46.076015Z","iopub.execute_input":"2024-04-14T16:08:46.076353Z","iopub.status.idle":"2024-04-14T16:08:46.121167Z","shell.execute_reply.started":"2024-04-14T16:08:46.076326Z","shell.execute_reply":"2024-04-14T16:08:46.120265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train_red['Label']\nX_train = train_red.drop('Label', axis=1)","metadata":{"id":"nOVi3srYxdTA","execution":{"iopub.status.busy":"2024-04-14T16:08:46.128417Z","iopub.execute_input":"2024-04-14T16:08:46.128813Z","iopub.status.idle":"2024-04-14T16:08:46.142833Z","shell.execute_reply.started":"2024-04-14T16:08:46.128788Z","shell.execute_reply":"2024-04-14T16:08:46.1418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGBOOST","metadata":{"id":"3TNUi6S8rDmk"}},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:08:46.149784Z","iopub.execute_input":"2024-04-14T16:08:46.1501Z","iopub.status.idle":"2024-04-14T16:08:46.19247Z","shell.execute_reply.started":"2024-04-14T16:08:46.150078Z","shell.execute_reply":"2024-04-14T16:08:46.191493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XGB = GradientBoostingClassifier(n_estimators=200, loss='log_loss', max_depth=4, ccp_alpha=0.0007).fit(X_train, y_train)\nprint(XGB.score(X_train, y_train))\nprint(classification_report(y_train, XGB.predict(X_train), target_names=['CE', 'LAA']))","metadata":{"id":"pkOlXugJGSH6","outputId":"229c0b83-e271-4822-ad66-51290592ee09","execution":{"iopub.status.busy":"2024-04-14T16:08:46.19364Z","iopub.execute_input":"2024-04-14T16:08:46.193922Z","iopub.status.idle":"2024-04-14T16:10:44.824042Z","shell.execute_reply.started":"2024-04-14T16:08:46.193899Z","shell.execute_reply":"2024-04-14T16:10:44.823076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_weighted_mc_log_loss(y_train, XGB.predict_proba(X_train))","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:10:44.82525Z","iopub.execute_input":"2024-04-14T16:10:44.825531Z","iopub.status.idle":"2024-04-14T16:10:44.882975Z","shell.execute_reply.started":"2024-04-14T16:10:44.825507Z","shell.execute_reply":"2024-04-14T16:10:44.882169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pred","metadata":{}},{"cell_type":"code","source":"def parse_images(folder):\n    # get the full paths of the images\n    imgs = [os.path.join(folder, f) for f in os.listdir(folder)]\n\n    df = pd.DataFrame()\n    df['image_path'] = imgs\n    # remove extension, and have the id as first and last component, eg: 006388_0\n    df['image_id'] = df['image_path'].apply(lambda x: '_'.join(x.split('/')[-1].replace('.jpg', '').split('_')[:2]))\n    # remove extension, and have the instance_id as last component only, eg: 0, 1, ...\n    df['instance_id'] = df['image_path'].apply(lambda x: int(x.split('_')[-1].replace('.jpg', '')))\n\n    df = df.sort_values(['image_id', 'instance_id']).reset_index(drop=True)\n\n    return df\n\ndef merge_image_info(image_df, info_df):\n    return image_df.merge(info_df, on='image_id', how='left').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:33:42.262767Z","iopub.execute_input":"2024-04-17T16:33:42.263156Z","iopub.status.idle":"2024-04-17T16:33:42.270978Z","shell.execute_reply.started":"2024-04-17T16:33:42.263108Z","shell.execute_reply":"2024-04-17T16:33:42.270039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:33:44.522697Z","iopub.execute_input":"2024-04-17T16:33:44.523376Z","iopub.status.idle":"2024-04-17T16:33:44.531674Z","shell.execute_reply.started":"2024-04-17T16:33:44.523342Z","shell.execute_reply":"2024-04-17T16:33:44.53081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = merge_image_info(parse_images('/kaggle/working/test'), test_csv)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:33:45.470685Z","iopub.execute_input":"2024-04-17T16:33:45.47103Z","iopub.status.idle":"2024-04-17T16:33:45.503632Z","shell.execute_reply.started":"2024-04-17T16:33:45.471003Z","shell.execute_reply":"2024-04-17T16:33:45.502725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:33:48.956689Z","iopub.execute_input":"2024-04-17T16:33:48.957325Z","iopub.status.idle":"2024-04-17T16:33:48.970569Z","shell.execute_reply.started":"2024-04-17T16:33:48.957291Z","shell.execute_reply":"2024-04-17T16:33:48.969631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize PyRadiomics feature extractor\nextractor = featureextractor.RadiomicsFeatureExtractor()\n\n# Define your lower and upper threshold values for yellow and brown\nlower_yellow = np.array([20, 100, 100])\nupper_yellow = np.array([30, 255, 255])\nlower_orange = np.array([5, 100, 100]) \nupper_orange = np.array([20, 255, 255])\nlower_brown = np.array([20, 50, 20]) \nupper_brown = np.array([40, 255, 200]) \n\n\n# Define the HSV color range for brown\nlower_brown = np.array([10, 50, 20])  \nupper_brown = np.array([40, 255, 255])  \n\n\nlower_blue = np.array([90, 50, 50])\nlower_purple = np.array([120, 50, 50])\nupper_purple = np.array([150, 255, 255])\nlower_dark_purple = np.array([100, 50, 50])  \nupper_dark_purple = np.array([120, 255, 255]) \nl_blu5 = np.array([67, 50, 50]) \nu_blu5 = np.array([340, 100, 255]) \n# Define lower and upper threshold values for white in HSV\nlower_white = np.array([0, 0, 200])\nupper_white = np.array([179, 30, 255])\n\n# Initialize list to store PyRadiomics data for all tiles of all images\npyradiomics_data = []\n\n\nfor i in range(len(df)):  \n    stained_image = cv2.imread(df[\"image_path\"][i])\n    #stained_image = np.array(tile)\n    hsv_image = cv2.cvtColor(stained_image, cv2.COLOR_BGR2HSV)\n\n    # RBC\n    orange_mask = cv2.inRange(hsv_image, lower_orange, upper_orange)\n    yellow_mask = cv2.inRange(hsv_image, lower_yellow, upper_yellow)\n    brown_mask = cv2.inRange(hsv_image, lower_brown, upper_brown)\n    yellow_brown_mask = cv2.bitwise_or(yellow_mask, brown_mask)\n    yellow_brown_mask = cv2.bitwise_or(yellow_brown_mask, orange_mask)\n\n    ########\n    rbc_mask = yellow_brown_mask.copy()\n    rbc_mask[rbc_mask > 0] = 1\n\n    # WBC\n  \n    dark_purple_mask = cv2.inRange(hsv_image, lower_dark_purple, upper_dark_purple)\n\n    blue_mask   = cv2.inRange(hsv_image, lower_blue, upper_purple)\n    purple_mask = cv2.inRange(hsv_image, lower_purple, upper_purple)\n    blue_purple_mask = cv2.bitwise_or(blue_mask, purple_mask)\n    purple_mask = cv2.bitwise_or(blue_purple_mask, dark_purple_mask)\n    pur_mask = cv2.inRange(hsv_image, l_blu5, u_blu5)\n    blue_purple_mask = cv2.bitwise_or(purple_mask, pur_mask)\n    \n    wbc_mask = blue_purple_mask.copy()\n    wbc_mask[wbc_mask > 0] = 1\n\n\n            # Fibrin/Platelets\n    white_mask = cv2.inRange(hsv_image, lower_white, upper_white)\n    fibrin_platelets_mask = cv2.bitwise_and(cv2.bitwise_not(yellow_brown_mask), cv2.bitwise_not(blue_purple_mask))\n    fibrin_platelets_mask = cv2.bitwise_and(fibrin_platelets_mask, cv2.bitwise_not(white_mask))\n    fibrin_platelets_mask[fibrin_platelets_mask > 0] = 1  # Ensure all non-zero values are set to 1\n\n\n    RBC = cv2.bitwise_and(stained_image, stained_image, mask = yellow_brown_mask)\n    WBC = cv2.bitwise_and(stained_image, stained_image, mask = blue_purple_mask)\n    FP  = cv2.bitwise_and(stained_image, stained_image, mask = fibrin_platelets_mask)\n\n    try:\n        # Read original image\n        original_image_sitk = sitk.GetImageFromArray(cv2.cvtColor(stained_image, cv2.COLOR_RGB2GRAY))\n\n        # Convert mask images to SimpleITK format\n        maskRBC_image_sitk = sitk.GetImageFromArray(rbc_mask)\n        maskWBC_image_sitk = sitk.GetImageFromArray(wbc_mask)\n        maskFP_image_sitk = sitk.GetImageFromArray(fibrin_platelets_mask)\n\n        # Extract features using PyRadiomics\n        featuresRBC = extractor.execute(original_image_sitk, maskRBC_image_sitk)\n        featuresWBC = extractor.execute(original_image_sitk, maskWBC_image_sitk)\n        featuresFP = extractor.execute(original_image_sitk, maskFP_image_sitk)\n\n        # Append data to list\n        pyradiomics_data.append({\n            'Image_ID': f\"{df['image_id'][i]}_{df['instance_id'][i]}\",\n            **{f\"RBC_{k}\": v for k, v in featuresRBC.items()},\n            **{f\"WBC_{k}\": v for k, v in featuresWBC.items()},\n            **{f\"FP_{k}\": v for k, v in featuresFP.items()}\n        })\n    except Exception as e:\n        print(f\"{i})Error extracting features for tile {df['instance_id'][i]} of image {df['image_id'][i]}: {e}\")\n        continue\n\n# Convert list of dictionaries to DataFrame\npyradiomics_df = pd.DataFrame(pyradiomics_data)\npyradiomics_df.to_csv('/kaggle/working/pyrad.csv')\nprint(\"All PyRadiomics features saved successfully.\")","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:13:16.167084Z","iopub.execute_input":"2024-04-14T16:13:16.167379Z","iopub.status.idle":"2024-04-14T16:16:23.532045Z","shell.execute_reply.started":"2024-04-14T16:13:16.167354Z","shell.execute_reply":"2024-04-14T16:16:23.531053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pyradiomics_df = pd.read_csv('/kaggle/working/pyrad.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.533297Z","iopub.execute_input":"2024-04-14T16:16:23.533611Z","iopub.status.idle":"2024-04-14T16:16:23.559886Z","shell.execute_reply.started":"2024-04-14T16:16:23.533584Z","shell.execute_reply":"2024-04-14T16:16:23.558876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pyradiomics_df = pyradiomics_df.drop(columns=[col for col in pyradiomics_df.columns if col not in significant_features and col not in [\"Image_ID\", \"Label\"]])","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.561499Z","iopub.execute_input":"2024-04-14T16:16:23.561874Z","iopub.status.idle":"2024-04-14T16:16:23.568888Z","shell.execute_reply.started":"2024-04-14T16:16:23.561841Z","shell.execute_reply":"2024-04-14T16:16:23.567866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = pyradiomics_df['Image_ID']\nX_test = pyradiomics_df.drop('Image_ID', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.570217Z","iopub.execute_input":"2024-04-14T16:16:23.57118Z","iopub.status.idle":"2024-04-14T16:16:23.58081Z","shell.execute_reply.started":"2024-04-14T16:16:23.571132Z","shell.execute_reply":"2024-04-14T16:16:23.579803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_prefix(image_id):\n    return '_'.join(image_id.split('_')[:-2])\n\nids = ids.apply(extract_prefix)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.581904Z","iopub.execute_input":"2024-04-14T16:16:23.582274Z","iopub.status.idle":"2024-04-14T16:16:23.592862Z","shell.execute_reply.started":"2024-04-14T16:16:23.582247Z","shell.execute_reply":"2024-04-14T16:16:23.591685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = X_test.replace(np.nan, 0)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.594201Z","iopub.execute_input":"2024-04-14T16:16:23.594479Z","iopub.status.idle":"2024-04-14T16:16:23.60339Z","shell.execute_reply.started":"2024-04-14T16:16:23.594454Z","shell.execute_reply":"2024-04-14T16:16:23.602453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.604769Z","iopub.execute_input":"2024-04-14T16:16:23.605153Z","iopub.status.idle":"2024-04-14T16:16:23.621551Z","shell.execute_reply.started":"2024-04-14T16:16:23.605119Z","shell.execute_reply":"2024-04-14T16:16:23.620405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.array(XGB.predict_proba(X_test))","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.622741Z","iopub.execute_input":"2024-04-14T16:16:23.623126Z","iopub.status.idle":"2024-04-14T16:16:23.632089Z","shell.execute_reply.started":"2024-04-14T16:16:23.623092Z","shell.execute_reply":"2024-04-14T16:16:23.631138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = ids.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.633262Z","iopub.execute_input":"2024-04-14T16:16:23.633572Z","iopub.status.idle":"2024-04-14T16:16:23.642499Z","shell.execute_reply.started":"2024-04-14T16:16:23.633537Z","shell.execute_reply":"2024-04-14T16:16:23.641686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = pd.DataFrame(np.concatenate((ids.reshape(-1,1), preds), axis=1), columns = ['patient_id', 'CE', 'LAA'])","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.643628Z","iopub.execute_input":"2024-04-14T16:16:23.643977Z","iopub.status.idle":"2024-04-14T16:16:23.65378Z","shell.execute_reply.started":"2024-04-14T16:16:23.643951Z","shell.execute_reply":"2024-04-14T16:16:23.652888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.groupby('patient_id').mean()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.654964Z","iopub.execute_input":"2024-04-14T16:16:23.655309Z","iopub.status.idle":"2024-04-14T16:16:23.668885Z","shell.execute_reply.started":"2024-04-14T16:16:23.655283Z","shell.execute_reply":"2024-04-14T16:16:23.667956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mce = preds['CE'].mean()\nmlaa = preds['LAA'].mean()","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:17:41.797921Z","iopub.execute_input":"2024-04-14T16:17:41.798884Z","iopub.status.idle":"2024-04-14T16:17:41.803802Z","shell.execute_reply.started":"2024-04-14T16:17:41.798851Z","shell.execute_reply":"2024-04-14T16:17:41.802718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for img in tooBig:\n    preds.loc['_'.join(img.split('_')[:-1])] = [mce, mlaa]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.67033Z","iopub.execute_input":"2024-04-14T16:16:23.67082Z","iopub.status.idle":"2024-04-14T16:16:23.679585Z","shell.execute_reply.started":"2024-04-14T16:16:23.670785Z","shell.execute_reply":"2024-04-14T16:16:23.678632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill NaN values with [0.5, 0.5] for 'CE' and 'LAA' columns\npreds.fillna({'CE': mce, 'LAA': mlaa}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.680814Z","iopub.execute_input":"2024-04-14T16:16:23.68113Z","iopub.status.idle":"2024-04-14T16:16:23.691497Z","shell.execute_reply.started":"2024-04-14T16:16:23.681106Z","shell.execute_reply":"2024-04-14T16:16:23.690503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"invalid_values_mask = (preds['CE'] > 1) | (preds['CE'] < 0) | (preds['LAA'] > 1) | (preds['LAA'] < 0)\npreds.loc[invalid_values_mask, ['CE', 'LAA']] = [mce, mlaa]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.692911Z","iopub.execute_input":"2024-04-14T16:16:23.6936Z","iopub.status.idle":"2024-04-14T16:16:23.703463Z","shell.execute_reply.started":"2024-04-14T16:16:23.693564Z","shell.execute_reply":"2024-04-14T16:16:23.702371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = '/kaggle/input/mayo-clinic-strip-ai/test'\nfiles = os.listdir(folder_path)\nfile_names = [os.path.splitext(file)[0] for file in files]\nmodified_list = ['_'.join(item.split('_')[:-1]) for item in file_names]","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.704686Z","iopub.execute_input":"2024-04-14T16:16:23.705028Z","iopub.status.idle":"2024-04-14T16:16:23.716106Z","shell.execute_reply.started":"2024-04-14T16:16:23.704977Z","shell.execute_reply":"2024-04-14T16:16:23.715253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_ids = preds.index\nfor patient_id in modified_list:\n    # Check if patient_id is not in the DataFrame index\n    if patient_id  not in pred_ids:\n        preds.loc[patient_id] = [mce, mlaa]\n        #print(\"0\")","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.718449Z","iopub.execute_input":"2024-04-14T16:16:23.718733Z","iopub.status.idle":"2024-04-14T16:16:23.723908Z","shell.execute_reply.started":"2024-04-14T16:16:23.718709Z","shell.execute_reply":"2024-04-14T16:16:23.722847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.reset_index()\npreds = preds.sort_values(by='patient_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.725292Z","iopub.execute_input":"2024-04-14T16:16:23.725636Z","iopub.status.idle":"2024-04-14T16:16:23.735555Z","shell.execute_reply.started":"2024-04-14T16:16:23.725609Z","shell.execute_reply":"2024-04-14T16:16:23.734656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.drop_duplicates(subset='patient_id', keep='first')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.736781Z","iopub.execute_input":"2024-04-14T16:16:23.737079Z","iopub.status.idle":"2024-04-14T16:16:23.748987Z","shell.execute_reply.started":"2024-04-14T16:16:23.737054Z","shell.execute_reply":"2024-04-14T16:16:23.748027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.750421Z","iopub.execute_input":"2024-04-14T16:16:23.751141Z","iopub.status.idle":"2024-04-14T16:16:23.758237Z","shell.execute_reply.started":"2024-04-14T16:16:23.751107Z","shell.execute_reply":"2024-04-14T16:16:23.757368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.to_csv('/kaggle/working/submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T16:16:23.75937Z","iopub.execute_input":"2024-04-14T16:16:23.759657Z","iopub.status.idle":"2024-04-14T16:16:23.770204Z","shell.execute_reply.started":"2024-04-14T16:16:23.759617Z","shell.execute_reply":"2024-04-14T16:16:23.769182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}