{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt \nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score \nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nimport warnings\nwarnings.filterwarnings('ignore')\nnp.random.seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:49.739233Z","iopub.execute_input":"2026-08-09T14:22:49.740131Z","iopub.status.idle":"2026-08-09T14:22:49.746689Z","shell.execute_reply.started":"2026-08-09T14:22:49.740087Z","shell.execute_reply":"2026-08-09T14:22:49.74504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df=pd.read_csv(\"/kaggle/input/competitions/rsna-knee-abnormality-detection/train.csv\")\ntrain_series_df = pd.read_csv(\"/kaggle/input/competitions/rsna-knee-abnormality-detection/train_series.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/competitions/rsna-knee-abnormality-detection/test.csv\")\ntest_series_df = pd.read_csv(\"/kaggle/input/competitions/rsna-knee-abnormality-detection/test_series.csv\")\nsample_sub = pd.read_csv(\"/kaggle/input/competitions/rsna-knee-abnormality-detection/sample_submission.csv\")\ntarget_labels=['ACL','MCL','Medial Meniscus', 'Lateral Meniscus', \n                 'Medial OA', 'Lateral OA', 'PF OA', 'Effusion', \n                 'Synovitis', \"Baker's\", 'Contusion', 'Fracture']\nprint(f\"Training samples: {len(train_df)}\")\nprint(f\"Test samples: {len(test_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:49.748579Z","iopub.execute_input":"2026-08-09T14:22:49.748942Z","iopub.status.idle":"2026-08-09T14:22:49.930157Z","shell.execute_reply.started":"2026-08-09T14:22:49.748915Z","shell.execute_reply":"2026-08-09T14:22:49.928911Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"def sigmoid(x):\n    x_clipped = np.clip(x, -500, 500)\n    return 1.0 / (1.0 + np.exp(-x_clipped))\ndef sigmoid_derivative(x):\n    s = sigmoid(x)\n    return s * (1.0 - s)\ndef tanh(x):\n    return np.tanh(x)\ndef tanh_derivative(x):\n    s = tanh(x)\n    return 1.0 - s**2\ndef relu(x):\n    return np.maximum(0, x)\ndef relu_derivative(x):\n    return (x > 0).astype(np.float64)\ndef leaky_relu(x, alpha=0.01):\n    return np.where(x > 0, x, alpha * x)\ndef leaky_relu_derivative(x, alpha=0.01):\n    return np.where(x > 0, 1.0, alpha)\ndef softmax(x):\n    x_stable = x - np.max(x, axis=1, keepdims=True)\n    exp_x = np.exp(x_stable)\n    return exp_x / np.sum(exp_x, axis=1, keepdims=True)\ndef binary_cross_entropy(y_true, y_pred):\n    y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7)\n    return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))\ndef binary_cross_entropy_derivative(y_true, y_pred):\n    y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7)\n    return -(y_true / y_pred) + (1 - y_true) / (1 - y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:49.931756Z","iopub.execute_input":"2026-08-09T14:22:49.932176Z","iopub.status.idle":"2026-08-09T14:22:49.944518Z","shell.execute_reply.started":"2026-08-09T14:22:49.932147Z","shell.execute_reply":"2026-08-09T14:22:49.943103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PCA:\n    def __init__(self,n_components=None):\n        self.n_components=n_components\n        self.components_=None\n        self.mean_=None\n        self.explained_variance_ratio=None\n        self.eigenvalues=None\n    def fit(self,x):\n        self.mean_=np.mean(x,axis=0)\n        x_centred=x-self.mean_\n        cov_matrix=np.cov(x_centred,rowvar=False)\n        eigenvalues,eigenvectors=np.linalg.eigh(cov_matrix)\n        idx=np.argsort(eigenvalues)[::-1]\n        eigenvalues=eigenvalues[idx]\n        eigenvectors=eigenvectors[:,idx]\n        if self.n_components is None: \n            self.n_components=x.shape[1]\n        self.components_=eigenvectors[:,:self.n_components]\n        self.eigenvalues_=eigenvalues[:self.n_components]\n        self.explained_variance_ratio=eigenvalues[:self.n_components]/np.sum(eigenvalues)\n        return self\n    def transform(self,x):\n        x_centred=x-self.mean_\n        return np.dot(x_centred,self.components_)\n    def fit_transform(self,x):\n        self.fit(x)\n        return self.transform(x)\n    def inverse_function(self,x_transformed):\n        return np.dot(x_transformed,self.components_.T)+self.mean_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:49.947141Z","iopub.execute_input":"2026-08-09T14:22:49.947492Z","iopub.status.idle":"2026-08-09T14:22:49.967223Z","shell.execute_reply.started":"2026-08-09T14:22:49.947465Z","shell.execute_reply":"2026-08-09T14:22:49.965773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Layer: \n    def __init__(self, input_dim,output_dim,activation='relu',dropout_rate=0.0):\n        if activation in ['relu','leaky_relu']:\n            scale=np.sqrt(2.0/input_dim)\n        else:\n            scale=np.sqrt(1.0/input_dim)\n        self.w=np.random.randn(input_dim,output_dim)*scale\n        self.b=np.zeros((1,output_dim))\n        self.activation=activation\n        self.dropout_rate=dropout_rate\n        self.m_w = np.zeros_like(self.w)\n        self.v_w = np.zeros_like(self.w)\n        self.m_b = np.zeros_like(self.b)\n        self.v_b = np.zeros_like(self.b)\n        self.input=None\n        self.z=None\n        self.output=None\n        self.dropout_mask=None\n    def forward(self,x,training=True):\n        self.input=x\n        self.z=np.dot(x,self.w)+self.b\n        if self.activation=='sigmoid':\n            self.output=sigmoid(self.z)\n        elif self.activation=='tanh':\n            self.output=tanh(self.z)\n        elif self.activation == 'relu':\n            self.output = relu(self.z)\n        elif self.activation == 'leaky_relu':\n            self.output = leaky_relu(self.z)\n        elif self.activation == 'linear':\n            self.output = self.z\n        elif self.activation == 'softmax':\n            self.output = softmax(self.z)\n        if training and self.dropout_rate>0:\n            self.dropout_mask=(np.random.rand(*self.output.shape)>self.dropout_rate)/(1-self.dropout_rate)\n            self.output*=self.dropout_mask\n        else:\n            self.dropout_mask=None\n        return self.output\n    def backward(self,d_output,learning_rate,beta1=0.9,beta2=0.999,epsilon=1e-8,t=1):\n        if self.dropout_mask is not None: \n            d_output*=self.dropout_mask\n        if self.activation == 'sigmoid':\n            d_z = d_output * sigmoid_derivative(self.z)\n        elif self.activation == 'tanh':\n            d_z = d_output * tanh_derivative(self.z)\n        elif self.activation == 'relu':\n            d_z = d_output * relu_derivative(self.z)\n        elif self.activation == 'leaky_relu':\n            d_z = d_output * leaky_relu_derivative(self.z)\n        elif self.activation == 'linear':\n            d_z = d_output\n        elif self.activation == 'softmax':\n            d_z = d_output\n        batch_size=self.input.shape[0]\n        d_w=np.dot(self.input.T,d_z)/batch_size\n        d_b=np.sum(d_z,axis=0,keepdims=True)/batch_size\n        d_input=np.dot(d_z,self.w.T)\n        self.m_w=beta1*self.m_w+(1-beta1)*d_w\n        self.v_w=beta2*self.v_w+(1-beta2)*(d_w**2)\n        m_w_hat=self.m_w/(1-beta1**t)\n        v_w_hat=self.v_w/(1-beta2**t)\n        self.w-=learning_rate*m_w_hat/(np.sqrt(v_w_hat)+epsilon)\n        self.m_b=beta1*self.m_b+(1-beta1)*d_b\n        self.v_b=beta2*self.v_b+(1-beta2)*(d_b**2)\n        m_b_hat=self.m_b/(1-beta1**t)\n        v_b_hat=self.v_b/(1-beta2**t)\n        self.b-=learning_rate*m_b_hat/(np.sqrt(v_b_hat)+epsilon)\n        return d_input","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:49.968786Z","iopub.execute_input":"2026-08-09T14:22:49.969429Z","iopub.status.idle":"2026-08-09T14:22:49.992297Z","shell.execute_reply.started":"2026-08-09T14:22:49.969383Z","shell.execute_reply":"2026-08-09T14:22:49.991455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Autoencoder:\n    def __init__(self,input_dim,encoding_dim=64,hidden_dims=[256,128]):\n        self.input_dim=input_dim\n        self.encoding_dim=encoding_dim\n        encoder_dim=[input_dim]+hidden_dims+[encoding_dim]\n        self.encoder_layers=[]\n        for i in range (len(encoder_dim)-1):\n            self.encoder_layers.append(Layer(encoder_dim[i],encoder_dim[i+1],activation='relu'))\n        decoder_dims=[encoding_dim]+hidden_dims[::-1]+[input_dim]\n        self.decoder_layers=[]\n        for i in range(len(decoder_dims)-2):\n            self.decoder_layers.append(Layer(decoder_dims[i],decoder_dims[i+1],activation='relu'))\n        self.decoder_layers.append(Layer(decoder_dims[-2],decoder_dims[-1],activation='relu'))\n        self.train_losses=[]\n        self.val_losses=[]\n    def encode(self,x,training=False):\n        h=x\n        for layer in self.encoder_layers:\n            h=layer.forward(h,training=training)\n        return h\n    def decode(self,z,training=False):\n        h=z\n        for layer in self.decoder_layers:\n            h=layer.forward(h,training=training)\n        return h\n    def forward(self,x,training=False):\n        encoded=self.encode(x,training=training)\n        decoded=self.decode(encoded,training=training)\n        return encoded,decoded\n    def compute_loss(self,x,x_reconstructed):\n        return np.mean((x-x_reconstructed)**2)\n    def backward(self,x,x_reconstructed,learning_rate,t):\n        batch_size=x.shape[0]\n        d_loss=-2*(x-x_reconstructed)/batch_size\n        d_h=d_loss\n        for layer in reversed(self.decoder_layers):\n            d_h=layer.backward(d_h,learning_rate,t=t)\n        for layer in reversed(self.encoder_layers):\n            d_h=layer.backward(d_h,learning_rate,t=t)\n    def train(self,x,x_val=None,epochs=100,batch_size=32,learning_rate=0.001,verbose=True):\n        n_samples=x.shape[0]\n        t=1\n        for epoch in range(epochs):\n            indices=np.random.permutation(n_samples)\n            epoch_loss=0\n            n_batches=0\n            for start in range(0,n_samples,batch_size):\n                end=min(start+batch_size,n_samples)\n                batch_idx=indices[start:end]\n                x_batch=x[batch_idx]\n                encoded,decoded=self.forward(x_batch,training=True)\n                loss=self.compute_loss(x_batch,decoded)\n                epoch_loss+=loss\n                n_batches+=1\n                self.backward(x_batch,decoded,learning_rate,t)\n                t+=1\n            avg_loss=epoch_loss/n_batches\n            self.train_losses.append(avg_loss)\n            if x_val is not None: \n                _,decoded_val=self.forward(x_val,training=False)\n                val_loss=self.compute_loss(x_val,decoded_val)\n                self.val_losses.append(val_loss)\n            if verbose and (epoch+1)%20==0:\n                val_str=f\", Val loss: {val_loss:.6f}\" if x_val is not None else \"\" \n                print(f\"Epoch {epoch+1}/{epochs},Loss:{avg_loss:.6f}{val_str}\")\n    def transform(self,x):\n        return self.encode(x,training=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:49.993734Z","iopub.execute_input":"2026-08-09T14:22:49.994161Z","iopub.status.idle":"2026-08-09T14:22:50.021635Z","shell.execute_reply.started":"2026-08-09T14:22:49.994123Z","shell.execute_reply":"2026-08-09T14:22:50.020329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NeuronNetwork:\n    def __init__(self, layer_dims, activations=None, dropout_rates=None, learning_rate=0.01, use_batch_norm=False):\n        self.layers = []\n        self.learning_rate = learning_rate\n        self.use_batch_norm = use_batch_norm\n        if activations is None:\n            activations = ['relu'] * (len(layer_dims) - 2) + ['sigmoid']\n        if dropout_rates is None:\n            dropout_rates = [0.0] * (len(layer_dims) - 1)\n        for i in range(len(layer_dims) - 1):\n            self.layers.append(Layer(layer_dims[i], layer_dims[i+1], activation=activations[i], dropout_rate=dropout_rates[i]))\n        self.train_losses = []\n        self.val_losses = []\n        self.train_auc = []\n        self.val_aucs = []\n    def forward(self, x, training=True):\n        h = x\n        for layer in self.layers:\n            h = layer.forward(h, training=training)\n        return h\n    def compute_loss(self, y_true, y_pred):\n        return binary_cross_entropy(y_true, y_pred)\n    def backward(self, y_true, y_pred, t):\n        d_loss = binary_cross_entropy_derivative(y_true, y_pred)\n        d_h = d_loss\n        for layer in reversed(self.layers):\n            d_h = layer.backward(d_h, self.learning_rate, t=t)\n    def train(self, x, y, x_val=None, y_val=None, epochs=100, batch_size=32, verbose=True, early_stopping_patience=10):\n        n_samples = x.shape[0]\n        t = 1\n        best_val_loss = float(\"inf\")\n        patience_counter = 0\n        best_weights = None\n        for epoch in range(epochs):\n            indices = np.random.permutation(n_samples)\n            epoch_loss = 0\n            n_batches = 0\n            for start in range(0, n_samples, batch_size):\n                end = min(start + batch_size, n_samples)\n                batch_idx = indices[start:end]\n                x_batch = x[batch_idx]\n                y_batch = y[batch_idx]\n                y_pred = self.forward(x_batch, training=True)\n                loss = self.compute_loss(y_batch, y_pred)\n                epoch_loss += loss\n                n_batches += 1\n                self.backward(y_batch, y_pred, t)\n                t += 1\n            avg_train_loss = epoch_loss / n_batches\n            self.train_losses.append(avg_train_loss)\n            if x_val is not None and y_val is not None:\n                y_val_pred = self.predict_proba(x_val)\n                val_loss = self.compute_loss(y_val, y_val_pred)\n                self.val_losses.append(val_loss)\n                try:\n                    val_auc = 0\n                    for i in range(y_val.shape[1]):\n                        val_auc += roc_auc_score(y_val[:, i], y_val_pred[:, i])\n                    val_auc /= y_val.shape[1]\n                    self.val_aucs.append(val_auc)\n                except:\n                    val_auc = 0.5\n                if val_loss < best_val_loss:\n                    best_val_loss = val_loss\n                    patience_counter = 0\n                    best_weights = [(layer.w.copy(), layer.b.copy()) for layer in self.layers]\n                else:\n                    patience_counter += 1\n                if patience_counter >= early_stopping_patience:\n                    if verbose:\n                        print(f\"Early stopping at {epoch+1}\")\n                    if best_weights is not None:\n                        for i, layer in enumerate(self.layers):\n                            layer.w, layer.b = best_weights[i]\n                    break\n                if verbose and (epoch + 1) % 20 == 0:\n                    val_str = f\", Val Loss: {val_loss:.4f}, Val AUC: {val_auc:.4f}\" if x_val is not None else \"\"\n                    print(f\"  Epoch {epoch+1}/{epochs}, Train Loss: {avg_train_loss:.4f}{val_str}\")\n        if best_weights is not None and patience_counter < early_stopping_patience:\n            for i, layer in enumerate(self.layers):\n                layer.w, layer.b = best_weights[i]\n    def predict_proba(self, X):\n        return self.forward(X, training=False)\n    def predict(self, X, threshold=0.5):\n        return (self.predict_proba(X) >= threshold).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:50.023029Z","iopub.execute_input":"2026-08-09T14:22:50.023598Z","iopub.status.idle":"2026-08-09T14:22:50.052032Z","shell.execute_reply.started":"2026-08-09T14:22:50.023545Z","shell.execute_reply":"2026-08-09T14:22:50.050791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['Report'] = train_df['Report'].fillna('')\ntfidf = TfidfVectorizer(max_features=500, stop_words='english', ngram_range=(1, 3), min_df=5, max_df=0.8)\ntrain_tfidf = tfidf.fit_transform(train_df['Report']).toarray()\npca = PCA(n_components=50)\ntext_pca = pca.fit_transform(train_tfidf)\nseries_agg = train_series_df.groupby('StudyInstanceUID').agg({'Fluid_Sensitive': ['mean', 'sum'], 'Fat_Suppression': ['mean', 'sum'], 'SeriesInstanceUID': 'count'}).reset_index()\nseries_agg.columns = ['StudyInstanceUID', 'fluid_mean', 'fluid_sum', 'fat_mean', 'fat_sum', 'num_series']\nplane_dummies = pd.get_dummies(train_series_df['Anatomical_Plane'], prefix='plane')\nplane_agg = plane_dummies.groupby(train_series_df['StudyInstanceUID']).mean()\nplane_agg.columns = [f'{col}_ratio' for col in plane_agg.columns]\nfeature_df = train_df[['StudyInstanceUID']].copy()\nfeature_df = feature_df.merge(series_agg, on='StudyInstanceUID', how='left')\nfeature_df = feature_df.merge(plane_agg, on='StudyInstanceUID', how='left')\nfor i in range(50):\n    feature_df[f'pca_{i}'] = text_pca[:, i]\nfeature_df = feature_df.fillna(0)\nfeature_cols = [c for c in feature_df.columns if c != 'StudyInstanceUID']\nprint(f\"Total features: {len(feature_cols)}\")\nnp.random.seed(42)\nae_scaler = StandardScaler()\nfeatures_scaled = ae_scaler.fit_transform(feature_df[feature_cols].values)\nnp.random.shuffle(features_scaled)\nsplit_idx = int(0.8 * len(features_scaled))\nX_train_ae = features_scaled[:split_idx]\nX_val_ae = features_scaled[split_idx:]\nsplit_idx = int(0.8 * len(features_scaled))\nX_train_ae = features_scaled[:split_idx]\nX_val_ae = features_scaled[split_idx:]\nprint(f\"Autoencoder input dim: {X_train_ae.shape[1]}\")\nautoencoder = Autoencoder(\n    input_dim=58,\n    encoding_dim=32,\n    hidden_dims=[64, 32]\n)\nautoencoder.train(X_train_ae, X_val_ae, epochs=500, batch_size=64, learning_rate=0.001)\nencoded_features = autoencoder.transform(features_scaled)\nprint(f\"Encoded features shape: {encoded_features.shape}\")\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\naxes[0].plot(autoencoder.train_losses, label='Train Loss', linewidth=2)\nif autoencoder.val_losses:\n    axes[0].plot(autoencoder.val_losses, label='Val Loss', linewidth=2)\naxes[0].set_xlabel('Epoch')\naxes[0].set_ylabel('Reconstruction Loss')\naxes[0].set_title('Autoencoder Training')\naxes[0].legend()\naxes[0].grid(True, alpha=0.3)\naxes[1].scatter(text_pca[:, 0], text_pca[:, 1], alpha=0.3, s=5)\naxes[1].set_xlabel(f'PC1 ({pca.explained_variance_ratio[0]:.2%})')\naxes[1].set_ylabel(f'PC2 ({pca.explained_variance_ratio[1]:.2%})')\naxes[1].set_title('Text Features (First 2 PCs)')\naxes[1].grid(True, alpha=0.3)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:22:50.053555Z","iopub.execute_input":"2026-08-09T14:22:50.05415Z","iopub.status.idle":"2026-08-09T14:23:41.253758Z","shell.execute_reply.started":"2026-08-09T14:22:50.054108Z","shell.execute_reply":"2026-08-09T14:23:41.252785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train = train_df[target_labels].values\ny_train_clean = np.nan_to_num(y_train, nan=0)\nX_train = features_scaled[:split_idx]\ny_train_split = y_train_clean[:split_idx]\nX_val = features_scaled[split_idx:]\ny_val = y_train_clean[split_idx:]\nclassifier = NeuronNetwork(\n    layer_dims=[58, 128, 64, 32, len(target_labels)],\n    activations=['relu', 'relu', 'relu', 'sigmoid'],\n    dropout_rates=[0.3, 0.3, 0.3, 0.0],\n    learning_rate=0.001\n)\nclassifier.train(\n    X_train, y_train_split,\n    X_val, y_val,\n    epochs=200,\n    batch_size=64,\n    verbose=True,\n    early_stopping_patience=20\n)\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\naxes[0].plot(classifier.train_losses, label='Train Loss', linewidth=2)\naxes[0].plot(classifier.val_losses, label='Validation Loss', linewidth=2)\naxes[0].set_xlabel('Epoch')\naxes[0].set_ylabel('Loss')\naxes[0].set_title('Classifier Training Progress')\naxes[0].legend()\naxes[0].grid(True, alpha=0.3)\nif classifier.val_aucs:\n    axes[1].plot(classifier.val_aucs, label='Validation AUC', linewidth=2, color='green')\n    axes[1].set_xlabel('Epoch')\n    axes[1].set_ylabel('AUC')\n    axes[1].set_title('Validation AUC')\n    axes[1].legend()\n    axes[1].grid(True, alpha=0.3)\n    axes[1].axhline(y=0.5, color='red', linestyle='--', alpha=0.3)\nplt.tight_layout()\nplt.show()\nif classifier.val_aucs:\n    print(f\"\\nBest Validation AUC: {max(classifier.val_aucs):.4f}\")\n    print(f\"Best Validation Loss: {min(classifier.val_losses):.4f}\")\ntest_series_agg = test_series_df.groupby('StudyInstanceUID').agg({\n    'Fluid_Sensitive': ['mean', 'sum'],\n    'Fat_Suppression': ['mean', 'sum'],\n    'SeriesInstanceUID': 'count'\n}).reset_index()\ntest_series_agg.columns = ['StudyInstanceUID', 'fluid_mean', 'fluid_sum', \n                          'fat_mean', 'fat_sum', 'num_series']\ntest_plane_dummies = pd.get_dummies(test_series_df['Anatomical_Plane'], prefix='plane')\ntest_plane_agg = test_plane_dummies.groupby(test_series_df['StudyInstanceUID']).mean()\ntest_plane_agg.columns = [f'{col}_ratio' for col in test_plane_agg.columns]\ntest_feature_df = test_df[['StudyInstanceUID']].copy()\ntest_feature_df = test_feature_df.merge(test_series_agg, on='StudyInstanceUID', how='left')\ntest_feature_df = test_feature_df.merge(test_plane_agg, on='StudyInstanceUID', how='left')\nif 'Report' in test_df.columns:\n    test_text = test_df['Report'].fillna('')\n    test_tfidf = tfidf.transform(test_text).toarray()\n    test_text_pca = pca.transform(test_tfidf)\nelse:\n    test_text_pca = np.zeros((len(test_df), 50))\nfor i in range(50):\n    test_feature_df[f'pca_{i}'] = test_text_pca[:, i]\ntest_feature_df = test_feature_df.fillna(0)\ntest_features_scaled = ae_scaler.transform(test_feature_df[feature_cols].values)\nprint(f\"Test features shape: {test_features_scaled.shape}\")\ntest_pred_proba = classifier.predict_proba(test_features_scaled)\nif len(test_pred_proba.shape) == 1:\n    test_pred_proba = test_pred_proba.reshape(-1, len(target_labels))\nprint(f\"Raw predictions shape: {test_pred_proba.shape}\")\nprint(f\"Raw predictions sample: {test_pred_proba[:2]}\")\ntest_pred_proba = np.nan_to_num(test_pred_proba, nan=0.5)\ntest_pred_proba = np.clip(test_pred_proba, 0.001, 0.999)\nprint(f\"Predictions stats: min={test_pred_proba.min():.4f}, max={test_pred_proba.max():.4f}\")\nsubmission = pd.DataFrame()\nsubmission['StudyInstanceUID'] = test_df['StudyInstanceUID'].values\nfor i, label in enumerate(target_labels):\n    submission[label] = test_pred_proba[:, i]\nsubmission.to_csv('submission.csv', index=False)\nsample_sub = pd.read_csv(\"/kaggle/input/competitions/rsna-knee-abnormality-detection/sample_submission.csv\")\nsub = pd.read_csv(\"submission.csv\")\nprint(\"\\nColumns match:\", all(sub.columns == sample_sub.columns))\nprint(\"Submission shape:\", sub.shape)\nprint(\"Sample shape:\", sample_sub.shape)\nprint(f\"NaN values: {sub[target_labels].isna().sum().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-09T14:25:04.290282Z","iopub.execute_input":"2026-08-09T14:25:04.29076Z","iopub.status.idle":"2026-08-09T14:25:08.141203Z","shell.execute_reply.started":"2026-08-09T14:25:04.290729Z","shell.execute_reply":"2026-08-09T14:25:08.140393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}