{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"> **Chargement des données**","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-03T11:06:47.12506Z","iopub.execute_input":"2023-12-03T11:06:47.125557Z","iopub.status.idle":"2023-12-03T11:06:58.709978Z","shell.execute_reply.started":"2023-12-03T11:06:47.125519Z","shell.execute_reply":"2023-12-03T11:06:58.708474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Spécification du chemin vers le dossier contenant les fichiers DICOM\ndata_path = '/kaggle/input/rsna-breast-cancer-detection'\n\n# Chargement des  métadonnées\ndf = pd.read_csv(os.path.join(data_path,'/kaggle/input/rsna-breast-cancer-detection/train.csv'))\n\n# Affichage  des premières lignes du métadonnées\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:06:58.712776Z","iopub.execute_input":"2023-12-03T11:06:58.713274Z","iopub.status.idle":"2023-12-03T11:06:58.838265Z","shell.execute_reply.started":"2023-12-03T11:06:58.713229Z","shell.execute_reply":"2023-12-03T11:06:58.837107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **normalisation des valeurs des pixels entre 0 et 1, et affichage des images originales et normalisées côte à côte.**\n\n* Mettre à l'échelle les valeurs des caractéristiques pour les ramener à une échelle commune. ","metadata":{}},{"cell_type":"code","source":"!pip install pydicom[gdcm] pylibjpeg","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:06:58.839868Z","iopub.execute_input":"2023-12-03T11:06:58.841082Z","iopub.status.idle":"2023-12-03T11:07:13.579147Z","shell.execute_reply.started":"2023-12-03T11:06:58.841031Z","shell.execute_reply":"2023-12-03T11:07:13.577265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install pylibjpeg pylibjpeg-libjpeg","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:07:13.582963Z","iopub.execute_input":"2023-12-03T11:07:13.583505Z","iopub.status.idle":"2023-12-03T11:07:28.41716Z","shell.execute_reply.started":"2023-12-03T11:07:13.583455Z","shell.execute_reply":"2023-12-03T11:07:28.414946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ndef normalize_images_for_patient(patient_id, directory_path):\n    patient_dir = os.path.join(directory_path, str(patient_id))\n\n    for filename in os.listdir(patient_dir):\n        file_path = os.path.join(patient_dir, filename)\n        dicom_data = pydicom.dcmread(file_path)\n        pixel_array = dicom_data.pixel_array\n\n        # Normaliser les valeurs des pixels entre 0 et 1\n        pixel_min = pixel_array.min()\n        pixel_max = pixel_array.max()\n        pixel_normalized = (pixel_array - pixel_min) / (pixel_max - pixel_min)\n\n        # Afficher l'image originale et normalisée pour illustration\n        plt.figure(figsize=(8, 4))\n        plt.subplot(1, 2, 1)\n        plt.imshow(pixel_array, cmap='gray')\n        plt.title('Image Originale')\n\n        plt.subplot(1, 2, 2)\n        plt.imshow(pixel_normalized, cmap='gray')\n        plt.title('Image Normalisée')\n\n        plt.show()\n\n# Spécifiez le chemin vers le répertoire contenant les fichiers DICOM (train_images)\ndirectory_path = '/kaggle/input/rsna-breast-cancer-detection/train_images'\n\n# Liste des patients\npatients = ['10006', '10011', '10025', '10038', '10042', '10048', '10049', '10050', '10051', '10086',\n            '10095', '10097', '10102', '10106', '10116', '10119', '10122', '10124', '10126', '10130',\n            '10132', '10136', '1014', '10144', '1015', '10151', '10152', '10153', '10175', '10179']\n\n# Normaliser les images pour chaque patient\nfor patient_id in patients:\n    normalize_images_for_patient(patient_id, directory_path)","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:07:28.419487Z","iopub.execute_input":"2023-12-03T11:07:28.41997Z","iopub.status.idle":"2023-12-03T11:08:30.516604Z","shell.execute_reply.started":"2023-12-03T11:07:28.419924Z","shell.execute_reply":"2023-12-03T11:08:30.511529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Nettoyage des données**\n\n* Gérer les valeurs manquantes en les supprimant, en les remplaçant par des valeurs appropriées (moyenne, médiane, etc.), ou en utilisant des techniques plus avancées.\n* Traiter les valeurs aberrantes qui peuvent affecter négativement le modèle.","metadata":{}},{"cell_type":"code","source":"df.isnull().sum().sum()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:07.92388Z","iopub.execute_input":"2023-12-03T11:09:07.924854Z","iopub.status.idle":"2023-12-03T11:09:07.969455Z","shell.execute_reply.started":"2023-12-03T11:09:07.924795Z","shell.execute_reply":"2023-12-03T11:09:07.967497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df)","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.408126Z","iopub.execute_input":"2023-12-03T11:09:08.409139Z","iopub.status.idle":"2023-12-03T11:09:08.418546Z","shell.execute_reply.started":"2023-12-03T11:09:08.409091Z","shell.execute_reply":"2023-12-03T11:09:08.416833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.456106Z","iopub.execute_input":"2023-12-03T11:09:08.457294Z","iopub.status.idle":"2023-12-03T11:09:08.488791Z","shell.execute_reply.started":"2023-12-03T11:09:08.45723Z","shell.execute_reply":"2023-12-03T11:09:08.486704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.491523Z","iopub.execute_input":"2023-12-03T11:09:08.492039Z","iopub.status.idle":"2023-12-03T11:09:08.522325Z","shell.execute_reply.started":"2023-12-03T11:09:08.491993Z","shell.execute_reply":"2023-12-03T11:09:08.520949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identification des valeurs manquantes dans le métadata\nmissing_values = df.isnull().sum()\n\n# Affichage des colonnes avec des valeurs manquantes et le nombre de valeurs manquantes\nprint(\"Valeurs manquantes par colonne :\")\nprint(missing_values[missing_values > 0])","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.529494Z","iopub.execute_input":"2023-12-03T11:09:08.529967Z","iopub.status.idle":"2023-12-03T11:09:08.557836Z","shell.execute_reply.started":"2023-12-03T11:09:08.529933Z","shell.execute_reply":"2023-12-03T11:09:08.556054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_statistics = df.groupby('cancer').describe()\nprint(class_statistics)","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.560294Z","iopub.execute_input":"2023-12-03T11:09:08.561965Z","iopub.status.idle":"2023-12-03T11:09:08.663484Z","shell.execute_reply.started":"2023-12-03T11:09:08.56189Z","shell.execute_reply":"2023-12-03T11:09:08.661874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#supprimer les lignes et les images dont la valeur de age est Nan\nimport os\nimport pandas as pd\n\n# Supprimer les lignes avec des valeurs NaN dans la colonne \"age\"\ndf = df.dropna(subset=['age'])","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.666397Z","iopub.execute_input":"2023-12-03T11:09:08.66681Z","iopub.status.idle":"2023-12-03T11:09:08.683906Z","shell.execute_reply.started":"2023-12-03T11:09:08.666778Z","shell.execute_reply":"2023-12-03T11:09:08.682239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.685431Z","iopub.execute_input":"2023-12-03T11:09:08.685981Z","iopub.status.idle":"2023-12-03T11:09:08.714693Z","shell.execute_reply.started":"2023-12-03T11:09:08.685929Z","shell.execute_reply":"2023-12-03T11:09:08.713436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ces données n'ont aucun impact sur le modéle donc il sont a supprimer\ndf=df.drop([\"site_id\" ,\"patient_id\" ,\"machine_id\" ] , axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.716894Z","iopub.execute_input":"2023-12-03T11:09:08.717288Z","iopub.status.idle":"2023-12-03T11:09:08.727729Z","shell.execute_reply.started":"2023-12-03T11:09:08.717253Z","shell.execute_reply":"2023-12-03T11:09:08.726679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Encodage des variables catégorielles**\n\n* Convertir les variables catégorielles en variables numériques.\n\n","metadata":{}},{"cell_type":"code","source":" #transformer les données catégorique en numérique\nfrom sklearn import preprocessing \nlabel_encoder = preprocessing.LabelEncoder() \ndf['laterality']= label_encoder.fit_transform(df['laterality']) \ndf['difficult_negative_case']= label_encoder.fit_transform(df['difficult_negative_case']) \ndf['density'] = df['density'].map({'A': 0 , 'B' : 1 ,'C': 2 , 'D' : 3})\ndf['view']= label_encoder.fit_transform(df['view']) \n# L =0 R=1\n# false = 0 true =1\n# A=0 B=1 C=2 D=3  bch tetfasa5 attention\n#At=0 CC=1 lm=2 lmo=3 ml=4 mlo=5","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.728875Z","iopub.execute_input":"2023-12-03T11:09:08.729241Z","iopub.status.idle":"2023-12-03T11:09:08.788761Z","shell.execute_reply.started":"2023-12-03T11:09:08.729209Z","shell.execute_reply":"2023-12-03T11:09:08.787449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.791854Z","iopub.execute_input":"2023-12-03T11:09:08.792264Z","iopub.status.idle":"2023-12-03T11:09:08.824146Z","shell.execute_reply.started":"2023-12-03T11:09:08.792231Z","shell.execute_reply":"2023-12-03T11:09:08.822572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns \n#utiliser le heatmap pour etudier les correlations\nplt.figure(figsize=(8, 8))\n\n#Définission la plage de valeurs à afficher sur la colormap de -1 à 1, et activez l'annotation pour afficher les valeurs de corrélation sur la heatmap.\nheatmap = sns.heatmap(df.corr(), vmin=-1, vmax=1, annot=True)\n\n#titre à la heatmap. La valeur de pad définit la distance entre le titre et le haut de la heatmap.\nheatmap.set_title('Correlation Heatmap', fontdict={'fontsize':12}, pad=12);","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:08.825746Z","iopub.execute_input":"2023-12-03T11:09:08.826101Z","iopub.status.idle":"2023-12-03T11:09:09.816484Z","shell.execute_reply.started":"2023-12-03T11:09:08.82607Z","shell.execute_reply":"2023-12-03T11:09:09.815588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selon le heatmap on a remarqué que density et cancer n'on pas une relation trés  forte entre eux alors \n# on a decidé de supprimés cette feature car elle contient plus que la moitié de la dataset des valeurs NaN ce qui influence sur les resultat\ndf.drop([\"density\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-03T11:09:09.818782Z","iopub.execute_input":"2023-12-03T11:09:09.820515Z","iopub.status.idle":"2023-12-03T11:09:09.846987Z","shell.execute_reply.started":"2023-12-03T11:09:09.820466Z","shell.execute_reply":"2023-12-03T11:09:09.845695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#on a remarqué que les deux classes ne sont pas repartie d'une facon equitable alors on\n# va utiliser le randomOverSampler pour dupliquer des données de la classe minoritaire.\nfrom imblearn.over_sampling import RandomOverSampler\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nx = df[['image_id','laterality','view','age','cancer','biopsy','invasive','BIRADS','implant','difficult_negative_case']]\ny = df['cancer']\n\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.30, random_state=60)\n\n# Instanciez le suréchantillonneur\noversampler = RandomOverSampler(random_state=42)\n\n# Appliquez le suréchantillonnage seulement sur l'ensemble d'entraînement\nx_train_resampled, y_train_resampled = oversampler.fit_resample(x_train, y_train)\n\n# Vérifiez la distribution des classes après le suréchantillonnage\nprint(pd.Series(y_train_resampled).value_counts())\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-03T12:13:54.706151Z","iopub.execute_input":"2023-12-03T12:13:54.707418Z","iopub.status.idle":"2023-12-03T12:13:54.764906Z","shell.execute_reply.started":"2023-12-03T12:13:54.707346Z","shell.execute_reply":"2023-12-03T12:13:54.763934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-12-03T12:13:36.95645Z","iopub.execute_input":"2023-12-03T12:13:36.957218Z","iopub.status.idle":"2023-12-03T12:13:36.968034Z","shell.execute_reply.started":"2023-12-03T12:13:36.957177Z","shell.execute_reply":"2023-12-03T12:13:36.966349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}