{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Chargement des données**","metadata":{}},{"cell_type":"code","source":"# Importer la librairie pandas pour la lecture des bases de données\nimport pandas as pd\n\n# Charger les bases de données\nX_train=pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\", sep=',')\nX_test=pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\", sep=',')\nsub=pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv\", sep=',')\n\n# Importer les librairies de visualisation de manipulation\nimport seaborn as sns\nimport matplotlib.pyplot as plt \nimport numpy as np\n\nimport os\nfrom tqdm import tqdm\nimport pydicom as dcm","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:10.76972Z","iopub.execute_input":"2023-03-01T14:06:10.770189Z","iopub.status.idle":"2023-03-01T14:06:12.059327Z","shell.execute_reply.started":"2023-03-01T14:06:10.770094Z","shell.execute_reply":"2023-03-01T14:06:12.05783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Information sur les données**","metadata":{}},{"cell_type":"code","source":"# Observer les 10 premières ligne du data frame\nX_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.061836Z","iopub.execute_input":"2023-03-01T14:06:12.062319Z","iopub.status.idle":"2023-03-01T14:06:12.095318Z","shell.execute_reply.started":"2023-03-01T14:06:12.062252Z","shell.execute_reply":"2023-03-01T14:06:12.093998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Afficher les résumés des informations sur les donnée\nprint(X_train.info())\nprint(X_test.info())","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.097031Z","iopub.execute_input":"2023-03-01T14:06:12.098056Z","iopub.status.idle":"2023-03-01T14:06:12.146347Z","shell.execute_reply.started":"2023-03-01T14:06:12.098002Z","shell.execute_reply":"2023-03-01T14:06:12.145312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decription du data frame \nX_train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.149489Z","iopub.execute_input":"2023-03-01T14:06:12.150696Z","iopub.status.idle":"2023-03-01T14:06:12.219183Z","shell.execute_reply.started":"2023-03-01T14:06:12.150641Z","shell.execute_reply":"2023-03-01T14:06:12.217713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"Dans certaines situations, il est préférable d'indexer la data frame sur le numéro du patient\n# Indexer les data frame avec la variable patient_id\ndef indexer (X):\n    X.set_index('patient_id', inplace=True, drop=True)\n    return X\nindexer(X_train)","metadata":{}},{"cell_type":"markdown","source":"# **Gestion des doublons**","metadata":{}},{"cell_type":"code","source":"X_train.duplicated().value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.22137Z","iopub.execute_input":"2023-03-01T14:06:12.221835Z","iopub.status.idle":"2023-03-01T14:06:12.264015Z","shell.execute_reply.started":"2023-03-01T14:06:12.221789Z","shell.execute_reply":"2023-03-01T14:06:12.262997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"On observe qu'il n'exite aucun doublon dans la base de données","metadata":{}},{"cell_type":"markdown","source":"# **Gestion des valeurs manquantes**","metadata":{}},{"cell_type":"code","source":"X_train.isnull().sum() # it shows how many missing values we have each column","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.265861Z","iopub.execute_input":"2023-03-01T14:06:12.266637Z","iopub.status.idle":"2023-03-01T14:06:12.288485Z","shell.execute_reply.started":"2023-03-01T14:06:12.266588Z","shell.execute_reply":"2023-03-01T14:06:12.287242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"Les variables age, BIRADS, density comptent respectivements 37, 28420 et 25236 valeurs manquantes. Retrouvons les meilleures astuces pour leur gestion.","metadata":{}},{"cell_type":"markdown","source":"## ***Observation sur la variable age***","metadata":{}},{"cell_type":"code","source":"# Observation de l'histogramme par rapport à l'age\nX_train.age.plot.hist()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.290344Z","iopub.execute_input":"2023-03-01T14:06:12.291064Z","iopub.status.idle":"2023-03-01T14:06:12.583434Z","shell.execute_reply.started":"2023-03-01T14:06:12.291022Z","shell.execute_reply":"2023-03-01T14:06:12.582052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observons le poids de l'âge sur la santé du patient par rapport au cancer\nplt.title(\"Poids de l'âge sur la santé du patient\")\nsns.histplot(x='age', data=X_train, kde=True, hue='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:12.587626Z","iopub.execute_input":"2023-03-01T14:06:12.588014Z","iopub.status.idle":"2023-03-01T14:06:13.467788Z","shell.execute_reply.started":"2023-03-01T14:06:12.587981Z","shell.execute_reply":"2023-03-01T14:06:13.466481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution sur la vairable age\nX_train.boxplot(column = \"age\")","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.469479Z","iopub.execute_input":"2023-03-01T14:06:13.469953Z","iopub.status.idle":"2023-03-01T14:06:13.68993Z","shell.execute_reply.started":"2023-03-01T14:06:13.469912Z","shell.execute_reply":"2023-03-01T14:06:13.688673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Définissons la fonction pour la déterminantion de la base comptant des valeurs manquantes selon une variable\ndef manquantes(X):\n    vmanq=X_train[X_train[X].isnull()]\n    return vmanq\n# Définissons la fonction pour la déterminantion de la base comptant des valeurs non-manquantes selon une variable\ndef non_manquantes(X):\n    vnmanq=X_train[X_train[X].notnull()]\n    return vnmanq","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.695315Z","iopub.execute_input":"2023-03-01T14:06:13.695737Z","iopub.status.idle":"2023-03-01T14:06:13.702789Z","shell.execute_reply.started":"2023-03-01T14:06:13.695698Z","shell.execute_reply":"2023-03-01T14:06:13.701326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Déterminons les valeurs manquantes par rapport à l'age\nnan_age=manquantes('age')\nnan_age","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.704243Z","iopub.execute_input":"2023-03-01T14:06:13.704697Z","iopub.status.idle":"2023-03-01T14:06:13.739764Z","shell.execute_reply.started":"2023-03-01T14:06:13.704659Z","shell.execute_reply":"2023-03-01T14:06:13.738421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"On constate qu'en grande partie, le numéro de patient apparait le plus souvent 4 fois exprimant le non renseignement de la variable âge de l'individu. Mais pour certaines données, on retrouve jusqu'à 5 fois le numéro du même patient, recherchons des semblables renseignés dans la base.","metadata":{}},{"cell_type":"code","source":"nan_age.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.741081Z","iopub.execute_input":"2023-03-01T14:06:13.741466Z","iopub.status.idle":"2023-03-01T14:06:13.762663Z","shell.execute_reply.started":"2023-03-01T14:06:13.741434Z","shell.execute_reply":"2023-03-01T14:06:13.761192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"On relève que pour chaque numéro de patient de la liste, l'âge est connu. Ainsi on peut completer dans la base après avoir relever les patient concernés par la situation. Pour ce faire, on introduit les lignes de codes pour appurement suivants dans le modèle de manipulation.\n\nX_train.loc[X_train['patient_id']==11995, 'age']=64\nX_train.loc[X_train['patient_id']==23752, 'age']=45\nX_train.loc[X_train['patient_id']==27212, 'age']=67\nX_train.loc[X_train['patient_id']==40791, 'age']=63\nX_train.loc[X_train['patient_id']==45891, 'age']=77\nX_train.loc[X_train['patient_id']==47764, 'age']=67\nX_train.loc[X_train['patient_id']==49020, 'age']=86\nX_train.loc[X_train['patient_id']==51500, 'age']=65","metadata":{}},{"cell_type":"code","source":"# Fonction d'observation sur la distibution des variables\ndef distrib(X): return X_train[X].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.764081Z","iopub.execute_input":"2023-03-01T14:06:13.764453Z","iopub.status.idle":"2023-03-01T14:06:13.773502Z","shell.execute_reply.started":"2023-03-01T14:06:13.76442Z","shell.execute_reply":"2023-03-01T14:06:13.77229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observation sur la variable density\ndistrib('density')","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.775263Z","iopub.execute_input":"2023-03-01T14:06:13.775656Z","iopub.status.idle":"2023-03-01T14:06:13.791457Z","shell.execute_reply.started":"2023-03-01T14:06:13.775621Z","shell.execute_reply":"2023-03-01T14:06:13.790219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_density=manquantes('density')\nnan_density.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.793126Z","iopub.execute_input":"2023-03-01T14:06:13.793515Z","iopub.status.idle":"2023-03-01T14:06:13.818856Z","shell.execute_reply.started":"2023-03-01T14:06:13.793479Z","shell.execute_reply":"2023-03-01T14:06:13.81771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"La moitié de l'ensemble des lignes comptent des valeurs manquante sur la variable density. Retrouvons des similarités dans l'autre partie.","metadata":{}},{"cell_type":"code","source":"exi_density=non_manquantes('density')\nexi_density.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.820381Z","iopub.execute_input":"2023-03-01T14:06:13.820757Z","iopub.status.idle":"2023-03-01T14:06:13.844329Z","shell.execute_reply.started":"2023-03-01T14:06:13.820722Z","shell.execute_reply":"2023-03-01T14:06:13.842924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observons la distribution selon la variable density\nf = sns.countplot(x=\"density\", data = X_train), plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:13.845997Z","iopub.execute_input":"2023-03-01T14:06:13.846489Z","iopub.status.idle":"2023-03-01T14:06:14.028697Z","shell.execute_reply.started":"2023-03-01T14:06:13.846441Z","shell.execute_reply":"2023-03-01T14:06:14.027386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Faisons varier continuelement les valeurs du patient_id appartenant au nan_patient_id pour retrouver \n# des semblabbles déjà renseignés\nvnan=nan_density['patient_id'].unique()\nprint('vnan= ', vnan)\nvexi=exi_density['patient_id'].unique()\nprint('vexi= ',vexi)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:14.030488Z","iopub.execute_input":"2023-03-01T14:06:14.031954Z","iopub.status.idle":"2023-03-01T14:06:14.041902Z","shell.execute_reply.started":"2023-03-01T14:06:14.031888Z","shell.execute_reply":"2023-03-01T14:06:14.040773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ordonnons ces 2 vecteurs \nvnan.sort()\nprint('vnan= ', vnan)\nvexi.sort()\nprint('vexi= ',vexi)\n\n# Vérifions l'existance de patients communs au 2\nfor i in vnan:\n    for j in vexi:\n        if i==j:\n            print(i)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:14.043267Z","iopub.execute_input":"2023-03-01T14:06:14.044371Z","iopub.status.idle":"2023-03-01T14:06:18.934067Z","shell.execute_reply.started":"2023-03-01T14:06:14.044334Z","shell.execute_reply":"2023-03-01T14:06:18.932768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"On constate qu'il n'existe aucune valeur commune dans ces vecteurs. Ainsi, les patients ne disposant pas de données sur la colonne 'density' ne présente pas d'information sur cette variable dans la base de données. Cela peut être observer sur la figure suivante. Il faut alors trouver un moyen de renseigner les lignes concernées par cette insuffisance.","metadata":{"execution":{"iopub.status.busy":"2023-02-20T09:42:13.975552Z","iopub.execute_input":"2023-02-20T09:42:13.975995Z","iopub.status.idle":"2023-02-20T09:42:18.432661Z","shell.execute_reply.started":"2023-02-20T09:42:13.975957Z","shell.execute_reply":"2023-02-20T09:42:18.431693Z"}}},{"cell_type":"code","source":"# Nous pouvons confirmer l'absence de valeur commun en observavnt sur une figure\n#(augmenter les dimension de la figure pour bien observer)\nfig, ax = plt.subplots(figsize=(10,5))\nax.plot(vnan)\nax.plot(vexi)\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:18.935594Z","iopub.execute_input":"2023-03-01T14:06:18.935969Z","iopub.status.idle":"2023-03-01T14:06:19.111759Z","shell.execute_reply.started":"2023-03-01T14:06:18.935933Z","shell.execute_reply":"2023-03-01T14:06:19.110502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Procédons de même pour BIRADS\nnan_birads=manquantes('BIRADS')\nexi_birads=non_manquantes('BIRADS')\n\nvnan=nan_birads['patient_id'].unique()\nprint('vnan= ', vnan)\nvexi=exi_birads['patient_id'].unique()\nprint('vexi= ',vexi)\n\n# Ordonnons ces 2 vecteurs \nvnan.sort()\nprint('vnan= ', vnan)\nvexi.sort()\nprint('vexi= ',vexi)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:19.113386Z","iopub.execute_input":"2023-03-01T14:06:19.113872Z","iopub.status.idle":"2023-03-01T14:06:19.137113Z","shell.execute_reply.started":"2023-03-01T14:06:19.113825Z","shell.execute_reply":"2023-03-01T14:06:19.1356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vérifions l'existance de patients communs au 2\nv=[]\nfor i in vnan:\n    for j in vexi:\n        if i==j:\n            v=v+[i]\nprint('le nombre de patients en commun dans les 2 vecteurs sont: ', len(v))","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:19.138824Z","iopub.execute_input":"2023-03-01T14:06:19.139635Z","iopub.status.idle":"2023-03-01T14:06:27.420535Z","shell.execute_reply.started":"2023-03-01T14:06:19.139583Z","shell.execute_reply":"2023-03-01T14:06:27.419175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"On relève de l'existence de patient dont le BIRADS non renseignés existe déjà dans une autre ligne du patient. Renseignons ces valeurs manquantes","metadata":{"execution":{"iopub.status.busy":"2023-02-21T14:47:34.984833Z","iopub.execute_input":"2023-02-21T14:47:34.98528Z","iopub.status.idle":"2023-02-21T14:47:42.605581Z","shell.execute_reply.started":"2023-02-21T14:47:34.985245Z","shell.execute_reply":"2023-02-21T14:47:42.604404Z"}}},{"cell_type":"code","source":"distrib('BIRADS')","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:27.421909Z","iopub.execute_input":"2023-03-01T14:06:27.422267Z","iopub.status.idle":"2023-03-01T14:06:27.435291Z","shell.execute_reply.started":"2023-03-01T14:06:27.422233Z","shell.execute_reply":"2023-03-01T14:06:27.433674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = sns.countplot(x=\"density\", data = X_train), plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:27.436905Z","iopub.execute_input":"2023-03-01T14:06:27.437296Z","iopub.status.idle":"2023-03-01T14:06:27.601782Z","shell.execute_reply.started":"2023-03-01T14:06:27.437243Z","shell.execute_reply":"2023-03-01T14:06:27.600586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Remplaçons les valeurs manquantes par les valeurs modales pour chacune ce 2 dans le modèle d'appurement en lui ajoutant les lignes suivants:\n\n\nX_train['BIRADS'].fillna(X_train['BIRADS'].mode().iloc[0], inplace=True)\n\n\nX_train['density'].fillna(X_train['density'].mode().iloc[0], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T09:45:20.712863Z","iopub.execute_input":"2023-02-27T09:45:20.713259Z","iopub.status.idle":"2023-02-27T09:45:20.729032Z","shell.execute_reply.started":"2023-02-27T09:45:20.713224Z","shell.execute_reply":"2023-02-27T09:45:20.727722Z"}}},{"cell_type":"code","source":"# Observation de valeurs manquantes dans le fichier de test\nX_test.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:27.603528Z","iopub.execute_input":"2023-03-01T14:06:27.6043Z","iopub.status.idle":"2023-03-01T14:06:27.616231Z","shell.execute_reply.started":"2023-03-01T14:06:27.604216Z","shell.execute_reply":"2023-03-01T14:06:27.614797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Exploitatation des données des fichiers DICOM**","metadata":{}},{"cell_type":"code","source":"# Exploitatation des données des fichiers DICOM\n# Nous définissons une fonction d'extraction de données à partir de fichiers au format DICOM\n# Un exemple de manipulation  est déjà fourni par participant\n\n# 4. Avons-nous des images pour toutes les entrées dans les données du train ?\n# Vérifions si chaque patient possède des images.\nfile_list = []\ntrain_path = \"/kaggle/input/rsna-breast-cancer-detection/train_images\"\nfolder_list = list(os.listdir(train_path))\nfor folder in tqdm(os.listdir(train_path)):\n    file_list += [x.split(\".dcm\")[0] for x in os.listdir(os.path.join(train_path, folder))]\nprint(len(folder_list), len(file_list))","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:06:27.617838Z","iopub.execute_input":"2023-03-01T14:06:27.618698Z","iopub.status.idle":"2023-03-01T14:07:28.508754Z","shell.execute_reply.started":"2023-03-01T14:06:27.618646Z","shell.execute_reply":"2023-03-01T14:07:28.507611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"diff = list(set(folder_list) - set([str(x) for x in X_train.patient_id.unique()]))\nprint(\"Differences in patient/folder list: \",len(diff))\ndiff = list(set(file_list) - set([str(x) for x in X_train.image_id.unique()]))\nprint(\"Differences in patient/folder list: \",len(diff))","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:07:28.510148Z","iopub.execute_input":"2023-03-01T14:07:28.51112Z","iopub.status.idle":"2023-03-01T14:07:28.590198Z","shell.execute_reply.started":"2023-03-01T14:07:28.51108Z","shell.execute_reply":"2023-03-01T14:07:28.588884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nous pouvons conclure que tous les patients et toutes les images sont indexés dans le jeu de données du train. Regardons ensuite la répartition de la latéralité et de la vue dans l'ensemble patient/images.","metadata":{}},{"cell_type":"code","source":"distrib('laterality')","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:07:28.598934Z","iopub.execute_input":"2023-03-01T14:07:28.599375Z","iopub.status.idle":"2023-03-01T14:07:28.611377Z","shell.execute_reply.started":"2023-03-01T14:07:28.599335Z","shell.execute_reply":"2023-03-01T14:07:28.610064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"!!!\nGlobalement, la plupart des images ont au moins une latéralité R et L ; selon la vue, la majorité présentera à la fois les vues MLO et CC, avec un petit groupe de patients ayant également 4 autres vues. Regroupons la latéralité et la vue pour obtenir toutes les combinaisons.","metadata":{}},{"cell_type":"code","source":"def extract_dicom_data(data_path, patient_id):\n    images_path = os.path.join(data_path,patient_id)\n    for image in os.listdir(images_path):\n        image_id = image.split(\".dcm\")[0]\n        image_path = os.path.join(images_path, image)\n        data_row_img_data = dcm.read_file(image_path)\n        print(\"=================================================\")\n        print(f\"Patient: {patient_id} Image_id: {image_id}\")\n        print(\"=================================================\")\n        print(data_row_img_data)\n        print(\"=================================================\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:07:28.61297Z","iopub.execute_input":"2023-03-01T14:07:28.614233Z","iopub.status.idle":"2023-03-01T14:07:28.623736Z","shell.execute_reply.started":"2023-03-01T14:07:28.614187Z","shell.execute_reply":"2023-03-01T14:07:28.622297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Et nous le testons sur un patient (avec 4 fichiers DICOM).\npatient_id = '10006'\nextract_dicom_data(train_path, patient_id)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:07:28.625394Z","iopub.execute_input":"2023-03-01T14:07:28.625819Z","iopub.status.idle":"2023-03-01T14:07:28.994953Z","shell.execute_reply.started":"2023-03-01T14:07:28.625783Z","shell.execute_reply":"2023-03-01T14:07:28.993645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Traitement des données DICOM\ndef process_dicom_data(data_path, patient_id, dicom_features):\n    images_path = os.path.join(data_path,str(patient_id))\n    for image in os.listdir(images_path):\n        try:\n            image_id = image.split(\".dcm\")[0]\n            image_path = os.path.join(images_path, image)\n            data_row_img_data = dcm.read_file(image_path)\n            rows = data_row_img_data.Rows\n            columns = data_row_img_data.Columns\n            content_date = data_row_img_data.ContentDate\n            photometric_interpretation = data_row_img_data.PhotometricInterpretation\n            dicom_features.append((image_id, rows, columns, content_date, photometric_interpretation))\n        except Exception as ex:\n            print(ex)\n            continue","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:07:28.996697Z","iopub.execute_input":"2023-03-01T14:07:28.997421Z","iopub.status.idle":"2023-03-01T14:07:29.006151Z","shell.execute_reply.started":"2023-03-01T14:07:28.997379Z","shell.execute_reply":"2023-03-01T14:07:29.004854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_features = []\nfor patient_id in tqdm(X_train.patient_id.unique()):\n    process_dicom_data(train_path, patient_id, dicom_features)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T14:07:29.007904Z","iopub.execute_input":"2023-03-01T14:07:29.008623Z","iopub.status.idle":"2023-03-01T15:11:45.745103Z","shell.execute_reply.started":"2023-03-01T14:07:29.00856Z","shell.execute_reply":"2023-03-01T15:11:45.740099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Exécutons cette fonction pour tous les fichiers.\nfeatures_train = pd.DataFrame(dicom_features)\nfeatures_train.columns = [\"image_id\", \"rows\", \"columns\", \"content_date\", \"photometric_interpretation\"]\nfeatures_train[\"image_id\"] = features_train[\"image_id\"].apply(lambda x: int(x))\nX_train = X_train.merge(features_train, on=\"image_id\")","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:45.752645Z","iopub.execute_input":"2023-03-01T15:11:45.753514Z","iopub.status.idle":"2023-03-01T15:11:46.04042Z","shell.execute_reply.started":"2023-03-01T15:11:45.75341Z","shell.execute_reply":"2023-03-01T15:11:46.03919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:46.042477Z","iopub.execute_input":"2023-03-01T15:11:46.042972Z","iopub.status.idle":"2023-03-01T15:11:46.074489Z","shell.execute_reply.started":"2023-03-01T15:11:46.042925Z","shell.execute_reply":"2023-03-01T15:11:46.072803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Nous avons déjà remarqué qu'il ne s'agit pas d'un nombre standard d'images ou d'une distribution de vue et de latéralité par patient. Voyons quelle est la répartition du nombre d'images/patient.\ntrain_agg_df = X_train.groupby([\"patient_id\"])[\"image_id\"].count().reset_index()\ntrain_agg_df.columns = [\"patient_id\", \"images\"]\ntrain_agg_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:46.077492Z","iopub.execute_input":"2023-03-01T15:11:46.078067Z","iopub.status.idle":"2023-03-01T15:11:46.103508Z","shell.execute_reply.started":"2023-03-01T15:11:46.077978Z","shell.execute_reply":"2023-03-01T15:11:46.102416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_agg_df.images.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:46.105158Z","iopub.execute_input":"2023-03-01T15:11:46.105542Z","iopub.status.idle":"2023-03-01T15:11:46.120071Z","shell.execute_reply.started":"2023-03-01T15:11:46.105507Z","shell.execute_reply":"2023-03-01T15:11:46.118822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Une majorité de 8 233 patients ont 4 images. Puis il y a de moins en moins de patients avec de plus en plus d'images. Seuls 2 patients ont 14 images.","metadata":{}},{"cell_type":"markdown","source":"# **Correlation entre les variables**","metadata":{}},{"cell_type":"code","source":"# Observons la corrélation entre les variables\ncorr_train = X_train.corr()\nplt.figure(figsize=(12, 7))\nsns.heatmap(corr_train, annot=True, fmt='.2g')","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:46.122027Z","iopub.execute_input":"2023-03-01T15:11:46.122403Z","iopub.status.idle":"2023-03-01T15:11:48.096349Z","shell.execute_reply.started":"2023-03-01T15:11:46.122369Z","shell.execute_reply":"2023-03-01T15:11:48.094837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Séparons les données en deux populations, les malades et les sains :\nmalade = X_train[X_train.cancer ==1]\nsain = X_train[X_train.cancer == 0]\n# Observons ces nouveaux vecteurs\nmalade.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.098228Z","iopub.execute_input":"2023-03-01T15:11:48.099658Z","iopub.status.idle":"2023-03-01T15:11:48.144557Z","shell.execute_reply.started":"2023-03-01T15:11:48.099603Z","shell.execute_reply":"2023-03-01T15:11:48.143367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.145982Z","iopub.execute_input":"2023-03-01T15:11:48.146388Z","iopub.status.idle":"2023-03-01T15:11:48.166607Z","shell.execute_reply.started":"2023-03-01T15:11:48.146353Z","shell.execute_reply":"2023-03-01T15:11:48.165262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fonction d'observation du poids d'une variable sur les états de santé des patients\ndef plot_hist(feature, bins = 20):\n    x1 = sain[feature].dropna()\n    x2 = malade[feature].dropna()\n    plt.hist([x1, x2], label=['Sain', 'Malade'], bins = bins)\n    plt.legend(loc = 'upper right')\n    plt.title('distribution relative de %s' %feature)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.16831Z","iopub.execute_input":"2023-03-01T15:11:48.168717Z","iopub.status.idle":"2023-03-01T15:11:48.181499Z","shell.execute_reply.started":"2023-03-01T15:11:48.168672Z","shell.execute_reply":"2023-03-01T15:11:48.180236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = sns.countplot(x=\"BIRADS\", data = X_train), plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.182888Z","iopub.execute_input":"2023-03-01T15:11:48.184071Z","iopub.status.idle":"2023-03-01T15:11:48.385619Z","shell.execute_reply.started":"2023-03-01T15:11:48.184029Z","shell.execute_reply":"2023-03-01T15:11:48.384325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_list_ts = []\ntest_path = \"/kaggle/input/rsna-breast-cancer-detection/test_images\"\nfolder_list_ts = list(os.listdir(test_path))\nfor folder in tqdm(os.listdir(test_path)):\n    file_list_ts += [x.split(\".dcm\")[0] for x in os.listdir(os.path.join(test_path, folder))]\nprint(len(folder_list_ts), len(file_list_ts))","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.387317Z","iopub.execute_input":"2023-03-01T15:11:48.388442Z","iopub.status.idle":"2023-03-01T15:11:48.412998Z","shell.execute_reply.started":"2023-03-01T15:11:48.38839Z","shell.execute_reply":"2023-03-01T15:11:48.411735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Et nous le testons sur un patient (avec 4 fichiers DICOM).\npatient_id = '10008'\nextract_dicom_data(test_path, patient_id)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.414641Z","iopub.execute_input":"2023-03-01T15:11:48.41526Z","iopub.status.idle":"2023-03-01T15:11:48.579705Z","shell.execute_reply.started":"2023-03-01T15:11:48.415221Z","shell.execute_reply":"2023-03-01T15:11:48.578365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_test_features = []\nfor patient_id in tqdm(X_test.patient_id.unique()):\n    process_dicom_data(test_path, patient_id, dicom_test_features)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.581592Z","iopub.execute_input":"2023-03-01T15:11:48.582122Z","iopub.status.idle":"2023-03-01T15:11:48.609696Z","shell.execute_reply.started":"2023-03-01T15:11:48.582063Z","shell.execute_reply":"2023-03-01T15:11:48.608421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Exécutons cette fonction pour tous les fichiers.\nfeatures_test_df = pd.DataFrame(dicom_test_features)\nfeatures_test_df.columns = [\"image_id\", \"rows\", \"columns\", \"content_date\", \"photometric_interpretation\"]\nfeatures_test_df[\"image_id\"] = features_test_df[\"image_id\"].apply(lambda x: int(x))\nX_test = X_test.merge(features_test_df, on=\"image_id\")","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.61157Z","iopub.execute_input":"2023-03-01T15:11:48.611927Z","iopub.status.idle":"2023-03-01T15:11:48.623665Z","shell.execute_reply.started":"2023-03-01T15:11:48.611895Z","shell.execute_reply":"2023-03-01T15:11:48.622227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.625227Z","iopub.execute_input":"2023-03-01T15:11:48.625846Z","iopub.status.idle":"2023-03-01T15:11:48.648972Z","shell.execute_reply.started":"2023-03-01T15:11:48.625807Z","shell.execute_reply":"2023-03-01T15:11:48.647639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Préparation de la Base de données d'entrainement**","metadata":{}},{"cell_type":"code","source":"# Fonction de préparation de la base de données d'entrainenment\n# pour la modélisation\ndef parse_train_model(X):\n    target=X.cancer\n    # Gestion des valeurs manquantes sur l'age des patients\n    X_train.loc[X_train['patient_id']==11995, 'age']=64\n    X_train.loc[X_train['patient_id']==23752, 'age']=45\n    X_train.loc[X_train['patient_id']==27212, 'age']=67\n    X_train.loc[X_train['patient_id']==40791, 'age']=63\n    X_train.loc[X_train['patient_id']==45891, 'age']=77\n    X_train.loc[X_train['patient_id']==47764, 'age']=67\n    X_train.loc[X_train['patient_id']==49020, 'age']=86\n    X_train.loc[X_train['patient_id']==51500, 'age']=65\n    # Gestion des valeurs manquantes sur les variables BIRADS et density\n    X_train['BIRADS'].fillna(X_train['BIRADS'].mode().iloc[0], inplace=True)\n    X_train['density'].fillna(X_train['density'].mode().iloc[0], inplace=True)\n    # Gestion (décomposition) de certaines variables catégorielles \n    to_dummy = ['laterality', 'view', 'density', 'photometric_interpretation']\n    for dum in to_dummy :\n        split_feature = pd.get_dummies(X[dum], prefix='split_'+dum)\n        X = X.join(split_feature)\n        del X[dum]\n    # Supression de certaines colonnes\n    to_del = ['cancer']\n    for col in to_del : del X[col]\n    return X, target","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.650831Z","iopub.execute_input":"2023-03-01T15:11:48.651198Z","iopub.status.idle":"2023-03-01T15:11:48.66421Z","shell.execute_reply.started":"2023-03-01T15:11:48.651164Z","shell.execute_reply":"2023-03-01T15:11:48.662747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fonction de préparation de la base de données d'entrainenment\n# pour la modélisation\nfrom sklearn.preprocessing import LabelEncoder\ndef parse_train_model_2(X):\n    target=X.cancer\n    # Gestion des valeurs manquantes sur l'age des patients\n    X_train.loc[X_train['patient_id']==11995, 'age']=64\n    X_train.loc[X_train['patient_id']==23752, 'age']=45\n    X_train.loc[X_train['patient_id']==27212, 'age']=67\n    X_train.loc[X_train['patient_id']==40791, 'age']=63\n    X_train.loc[X_train['patient_id']==45891, 'age']=77\n    X_train.loc[X_train['patient_id']==47764, 'age']=67\n    X_train.loc[X_train['patient_id']==49020, 'age']=86\n    X_train.loc[X_train['patient_id']==51500, 'age']=65\n    # Gestion des valeurs manquantes sur les variables BIRADS et density\n    X_train['BIRADS'].fillna(X_train['BIRADS'].mode().iloc[0], inplace=True)\n    X_train['density'].fillna(X_train['density'].mode().iloc[0], inplace=True)\n    # Gestion (décomposition) de certaines variables catégorielles \n    to_encod= ['laterality', 'view', 'density', 'photometric_interpretation']\n    le = LabelEncoder()\n    for labels in to_encod:\n        X.sort_values(by = labels) \n        le.fit(labels)\n        labels = le.transform(labels)\n        X = X.join(labels)\n        del X[labels]\n    # Supression de certaines colonnes\n    to_del = ['cancer']\n    for col in to_del : del X[col]\n    return X, target","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:11:48.666534Z","iopub.execute_input":"2023-03-01T15:11:48.666996Z","iopub.status.idle":"2023-03-01T15:11:48.858108Z","shell.execute_reply.started":"2023-03-01T15:11:48.666935Z","shell.execute_reply":"2023-03-01T15:11:48.856733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Préparation de la Base de données de test**","metadata":{}},{"cell_type":"code","source":"# Fonction de préparation de la base de données de test\n# pour la modélisation\ndef parse_test_model(X):\n    # Gestion des certaines variables catégorielles\n    to_dummy = ['laterality', 'view', 'prediction_id', 'photometric_interpretation']\n    for dum in to_dummy :\n        split_feature = pd.get_dummies(X[dum], prefix='split_'+dum)\n        X = X.join(split_feature)\n        del X[dum]\n    to_del = []\n    for col in to_del : del X[col]\n    return X#, target","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:08.086518Z","iopub.execute_input":"2023-03-01T15:18:08.087394Z","iopub.status.idle":"2023-03-01T15:18:08.098873Z","shell.execute_reply.started":"2023-03-01T15:18:08.087308Z","shell.execute_reply":"2023-03-01T15:18:08.095556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtest=parse_test_model(X_test.copy())","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:11.3235Z","iopub.execute_input":"2023-03-01T15:18:11.323933Z","iopub.status.idle":"2023-03-01T15:18:11.345032Z","shell.execute_reply.started":"2023-03-01T15:18:11.323896Z","shell.execute_reply":"2023-03-01T15:18:11.343906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtest.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:15.150458Z","iopub.execute_input":"2023-03-01T15:18:15.150927Z","iopub.status.idle":"2023-03-01T15:18:15.176188Z","shell.execute_reply.started":"2023-03-01T15:18:15.150888Z","shell.execute_reply":"2023-03-01T15:18:15.175011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Appel :\nxtrain, ytrain = parse_train_model(X_train)\n# La matrice X ainsi obtenue pour notre modèle 0\nxtrain.head(3)#X.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:18.619027Z","iopub.execute_input":"2023-03-01T15:18:18.619523Z","iopub.status.idle":"2023-03-01T15:18:18.734703Z","shell.execute_reply.started":"2023-03-01T15:18:18.619481Z","shell.execute_reply":"2023-03-01T15:18:18.732889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:21.898632Z","iopub.execute_input":"2023-03-01T15:18:21.899054Z","iopub.status.idle":"2023-03-01T15:18:21.92696Z","shell.execute_reply.started":"2023-03-01T15:18:21.89902Z","shell.execute_reply":"2023-03-01T15:18:21.925605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Validation croisée**","metadata":{}},{"cell_type":"code","source":"# Fonction d'entrainement de modèle\n# Evaluer un score par validation croisée.\nfrom sklearn.model_selection import cross_val_score\ndef compute_score(model, X, y):\n    res = cross_val_score(model, X, y, cv = 5, error_score='raise')\n    trnval= np.round(res.mean()*100,8)\n    return trnval","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:26.544816Z","iopub.execute_input":"2023-03-01T15:18:26.545234Z","iopub.status.idle":"2023-03-01T15:18:26.627246Z","shell.execute_reply.started":"2023-03-01T15:18:26.545201Z","shell.execute_reply":"2023-03-01T15:18:26.625823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fonction d'entrainement de modèle\n# Evaluer un score par validation croisée.\nfrom sklearn.model_selection import cross_validate\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import make_scorer, r2_score\n\ndef compute_score_detail(model, X, y):\n    res= cross_validate(model, X, y, cv=5, scoring=dict(r2=make_scorer(r2_score), e2=make_scorer(mean_squared_error)),\n              return_train_score=False)\n    return res","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:28.691435Z","iopub.execute_input":"2023-03-01T15:18:28.691878Z","iopub.status.idle":"2023-03-01T15:18:28.701026Z","shell.execute_reply.started":"2023-03-01T15:18:28.691842Z","shell.execute_reply":"2023-03-01T15:18:28.698853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nmodel_LR = LogisticRegression()#(max_iter=1000)\nprint(compute_score(model_LR, xtrain, ytrain))\ncompute_score_detail(model_LR, xtrain, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:31.228208Z","iopub.execute_input":"2023-03-01T15:18:31.228667Z","iopub.status.idle":"2023-03-01T15:18:38.371672Z","shell.execute_reply.started":"2023-03-01T15:18:31.22863Z","shell.execute_reply":"2023-03-01T15:18:38.370304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Modélisation**","metadata":{}},{"cell_type":"code","source":"# Fonction d'entrainement de la machine et de prédiction du modèle\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import make_scorer, r2_score\nfrom sklearn.model_selection import train_test_split\n\ndef prediction(model, X_train, X_test, y_train):\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, train_size=0.8)\n    #Xtrain, Xtest, y_train, y_test = train_test_split(X, y)\n    # Entrainement du modèle\n    model.fit(X_train,y_train)\n    # Test du modèle\n    predi=model.predict(X_test)\n    # Performance du modèle\n    performa=accuracy_score(y_test,predi); print('performance du modèle : ', model,'est: ', performa)\n    matrix=confusion_matrix(y_test,predi); print('Matrice de confusion du modèle ',model ,'est: ',); print(matrix)\n    y_test.to_csv('soumission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:38.374532Z","iopub.execute_input":"2023-03-01T15:18:38.375537Z","iopub.status.idle":"2023-03-01T15:18:38.396219Z","shell.execute_reply.started":"2023-03-01T15:18:38.37548Z","shell.execute_reply":"2023-03-01T15:18:38.394247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Modèle: Régression Logistique**","metadata":{}},{"cell_type":"code","source":"model_LR = LogisticRegression()#(max_iter=1000)\nprediction(model_LR, xtrain, xtest, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:39.449798Z","iopub.execute_input":"2023-03-01T15:18:39.450196Z","iopub.status.idle":"2023-03-01T15:18:40.247398Z","shell.execute_reply.started":"2023-03-01T15:18:39.450163Z","shell.execute_reply":"2023-03-01T15:18:40.243044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Modèle: Random Forest**","metadata":{}},{"cell_type":"code","source":"# Model RF\nfrom sklearn.ensemble import RandomForestClassifier\n# Création du modèle\nmodel_RF=RandomForestClassifier()\nprediction(model_RF, xtrain, xtest, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:18:46.826764Z","iopub.execute_input":"2023-03-01T15:18:46.82718Z","iopub.status.idle":"2023-03-01T15:18:50.075474Z","shell.execute_reply.started":"2023-03-01T15:18:46.827146Z","shell.execute_reply":"2023-03-01T15:18:50.074385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Naive Bayes**","metadata":{}},{"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\nmodel_GNB=GaussianNB()\nprediction(model_GNB, xtrain, xtest, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:19:14.655909Z","iopub.execute_input":"2023-03-01T15:19:14.656363Z","iopub.status.idle":"2023-03-01T15:19:14.884134Z","shell.execute_reply.started":"2023-03-01T15:19:14.656323Z","shell.execute_reply":"2023-03-01T15:19:14.881792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Arbre à décision**","metadata":{}},{"cell_type":"code","source":"from sklearn import tree\nmodel_DT= tree.DecisionTreeClassifier()\nprediction(model_DT, xtrain, xtest, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:19:17.356711Z","iopub.execute_input":"2023-03-01T15:19:17.357153Z","iopub.status.idle":"2023-03-01T15:19:17.609437Z","shell.execute_reply.started":"2023-03-01T15:19:17.357116Z","shell.execute_reply":"2023-03-01T15:19:17.607958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **SVM**","metadata":{}},{"cell_type":"code","source":"from sklearn.svm import LinearSVC\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nmodel_SVM = make_pipeline(StandardScaler(), LinearSVC(random_state=0, tol=1e-5, max_iter=1000))\nprediction(model_SVM, xtrain, xtest, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:19:19.829458Z","iopub.execute_input":"2023-03-01T15:19:19.829861Z","iopub.status.idle":"2023-03-01T15:19:38.726547Z","shell.execute_reply.started":"2023-03-01T15:19:19.829829Z","shell.execute_reply":"2023-03-01T15:19:38.725093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Conclusion**","metadata":{}},{"cell_type":"markdown","source":"En observant les différente figures produites après les modélisation et en considérant les perfermances des algorithmes, nous pouvons faire le classement par ordre de préférence suivant: **Random Forest**, **Arbre à décision**, **SVM**, **Random Forest**, **Naive Bayes** puis la **Régression Logistique**.\n\n\nCependant les différentes matrices de confusions produites laisse sans doute à négliger *Régression Logistique* de même que *Naive Bayes*.\n\n\nAinsi, on retient comme modèle de préfence:**Arbre à décision**, **SVM**, **Random Forest**,  puis le **Naive Bayes**.\n\nNous maintenons le **Radom Forest** comme modèle privlégié. Et on peut observer l'influence de chacune des variable sur le modèle dans la figure qui suit.\n\n","metadata":{}},{"cell_type":"code","source":"# Model RF\nfrom sklearn.ensemble import RandomForestClassifier\n# Création du modèle\nmodel_RF=RandomForestClassifier()\nprediction(model_RF, xtrain, xtest, ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:19:38.729312Z","iopub.execute_input":"2023-03-01T15:19:38.730183Z","iopub.status.idle":"2023-03-01T15:19:41.447158Z","shell.execute_reply.started":"2023-03-01T15:19:38.73013Z","shell.execute_reply":"2023-03-01T15:19:41.44572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fonction de détermination de l'influence de chaque variable sur le model\nimport pylab as pl\ndef clf_importance(X, clf):\n    importances = clf.feature_importances_\n    indices = np.argsort(importances)[::-1]\n    pl.title(\"Feature importances\")\n    for tree in clf.estimators_:\n        pl.plot(range(X.shape[1]), tree.feature_importances_[indices],\"r\")\n        pl.plot(range(X.shape[1]), importances[indices], \"b\")\n        pl.grid(True)\n    pl.show()\n        \n    for f in range(X.shape[1]):\n        print(\"%d. feature : %s (%f)\" % (f + 1, X.columns[indices[f]], importances[indices[f]]))","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:19:41.449111Z","iopub.execute_input":"2023-03-01T15:19:41.449548Z","iopub.status.idle":"2023-03-01T15:19:41.460984Z","shell.execute_reply.started":"2023-03-01T15:19:41.449511Z","shell.execute_reply":"2023-03-01T15:19:41.459165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importance de l'influence des variable sur le model\nmodel_RF.fit(xtrain, ytrain)\nclf_importance(xtrain, model_RF)","metadata":{"execution":{"iopub.status.busy":"2023-03-01T15:19:54.096471Z","iopub.execute_input":"2023-03-01T15:19:54.096886Z","iopub.status.idle":"2023-03-01T15:19:57.213796Z","shell.execute_reply.started":"2023-03-01T15:19:54.096852Z","shell.execute_reply":"2023-03-01T15:19:57.212339Z"},"trusted":true},"execution_count":null,"outputs":[]}]}