{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HyperbandSearchCV for hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport xgboost as xgb\nfrom sklearn.metrics import confusion_matrix, roc_auc_score, accuracy_score, f1_score, roc_curve\nfrom sklearn.experimental import enable_halving_search_cv\nfrom sklearn.model_selection import train_test_split, GridSearchCV, HalvingGridSearchCV\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nfrom copy import deepcopy\nimport time\nfrom sklearn.multioutput import MultiOutputClassifier\n\nmpl.rcParams['figure.dpi'] = 200","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-07T22:18:18.670244Z","iopub.execute_input":"2023-07-07T22:18:18.670628Z","iopub.status.idle":"2023-07-07T22:18:19.514578Z","shell.execute_reply.started":"2023-07-07T22:18:18.670598Z","shell.execute_reply":"2023-07-07T22:18:19.513581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_xgb = xgb.XGBClassifier(objective=\"binary:logistic\", random_state=42, tree_method = \"gpu_hist\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:19.516467Z","iopub.execute_input":"2023-07-07T22:18:19.516822Z","iopub.status.idle":"2023-07-07T22:18:19.525031Z","shell.execute_reply.started":"2023-07-07T22:18:19.516791Z","shell.execute_reply":"2023-07-07T22:18:19.524103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_xgb.load_model(\"/kaggle/input/xgbdata/xgboost-full-2905-5pm.json\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:19.546125Z","iopub.execute_input":"2023-07-07T22:18:19.546394Z","iopub.status.idle":"2023-07-07T22:18:36.556271Z","shell.execute_reply.started":"2023-07-07T22:18:19.546371Z","shell.execute_reply":"2023-07-07T22:18:36.555257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = np.load(\"/kaggle/input/xgbdata/Y_1499.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:36.558354Z","iopub.execute_input":"2023-07-07T22:18:36.558739Z","iopub.status.idle":"2023-07-07T22:18:45.335867Z","shell.execute_reply.started":"2023-07-07T22:18:36.558707Z","shell.execute_reply":"2023-07-07T22:18:45.334891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_to_consider = np.load(\"/kaggle/input/xgbdata/Y_1499_labels.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:45.337348Z","iopub.execute_input":"2023-07-07T22:18:45.337745Z","iopub.status.idle":"2023-07-07T22:18:45.346782Z","shell.execute_reply.started":"2023-07-07T22:18:45.337711Z","shell.execute_reply":"2023-07-07T22:18:45.345729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = '/kaggle/input/t5embeds/train_embeds.npy'","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:45.351918Z","iopub.execute_input":"2023-07-07T22:18:45.352193Z","iopub.status.idle":"2023-07-07T22:18:45.357671Z","shell.execute_reply.started":"2023-07-07T22:18:45.352169Z","shell.execute_reply":"2023-07-07T22:18:45.35662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(fn)\nX = np.load(fn)\nprint(X.shape)\nX","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:45.359063Z","iopub.execute_input":"2023-07-07T22:18:45.359692Z","iopub.status.idle":"2023-07-07T22:18:55.076252Z","shell.execute_reply.started":"2023-07-07T22:18:45.359653Z","shell.execute_reply":"2023-07-07T22:18:55.075324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_probabilities = np.load('/kaggle/input/cafa-5-submission/xgb-prediction-full-probabilities-06-06.npy')","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:18:55.077745Z","iopub.execute_input":"2023-07-07T22:18:55.078096Z","iopub.status.idle":"2023-07-07T22:19:00.334785Z","shell.execute_reply.started":"2023-07-07T22:18:55.078064Z","shell.execute_reply":"2023-07-07T22:19:00.333758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IX = np.arange(len(X))\nprint(IX.shape)\nprint(IX)\nIX_train, IX_test, _,_ = train_test_split( IX, IX, train_size=0.1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:19:00.336177Z","iopub.execute_input":"2023-07-07T22:19:00.336589Z","iopub.status.idle":"2023-07-07T22:19:00.351324Z","shell.execute_reply.started":"2023-07-07T22:19:00.336534Z","shell.execute_reply":"2023-07-07T22:19:00.350208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = X[IX_train[:10000],:], X[IX_test[:10000],:], Y[IX_train[:10000],:], Y[IX_test[:10000],:]","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:19:00.353065Z","iopub.execute_input":"2023-07-07T22:19:00.353479Z","iopub.status.idle":"2023-07-07T22:19:00.447141Z","shell.execute_reply.started":"2023-07-07T22:19:00.353416Z","shell.execute_reply":"2023-07-07T22:19:00.446156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperband Search CV implemnation\n#### Code used from the following repo - [https://github.com/thuijskens/scikit-hyperband](http://)","metadata":{}},{"cell_type":"markdown","source":"HyperBandSearchCV uses scikitlearn's BaseSearchCV as the base class  \nHyperBandSearchCV2 uses scikitlearn's GridSearchCV as the base class","metadata":{}},{"cell_type":"code","source":"import copy\n\nimport numpy as np\nfrom scipy.stats import rankdata\n\nfrom sklearn.utils import check_random_state\nfrom sklearn.model_selection._search import BaseSearchCV, ParameterSampler\n\n\n__all__ = ['HyperbandSearchCV']\n\n\nclass HyperbandSearchCV(BaseSearchCV):\n    def __init__(self, estimator, param_distributions,\n                 resource_param='n_estimators', eta=3, min_iter=1,\n                 max_iter=81, skip_last=0, scoring=None, n_jobs=1,\n                 refit=True, cv=None,\n                 verbose=0, pre_dispatch='2*n_jobs', random_state=None,\n                 error_score='raise', return_train_score=False):\n        self.param_distributions = param_distributions\n        self.resource_param = resource_param\n        self.eta = eta\n        self.min_iter = min_iter\n        self.max_iter = max_iter\n        self.skip_last = skip_last\n        self.random_state = random_state\n\n        super(HyperbandSearchCV, self).__init__(\n            estimator=estimator, scoring=scoring, n_jobs=n_jobs,\n            refit=refit, cv=cv, verbose=verbose,\n            pre_dispatch=pre_dispatch, error_score=error_score,\n            return_train_score=return_train_score)\n\n    def _run_search(self, evaluate_candidates):\n        self._validate_input()\n\n        s_max = int(np.floor(np.log(self.max_iter / self.min_iter) / np.log(self.eta)))\n        B = (s_max + 1) * self.max_iter\n\n        refit_metric = 'score'\n        random_state = check_random_state(self.random_state)\n\n        if self.skip_last > s_max:\n            raise ValueError('skip_last is higher than the total number of rounds')\n\n        for round_index, s in enumerate(reversed(range(s_max + 1))):\n            n = int(np.ceil(int(B / self.max_iter / (s + 1)) * np.power(self.eta, s)))\n\n            # initial number of iterations per config\n            r = self.max_iter / np.power(self.eta, s)\n            configurations = list(ParameterSampler(param_distributions=self.param_distributions,\n                                                   n_iter=n,\n                                                   random_state=random_state))\n\n            if self.verbose > 0:\n                print('Starting bracket {0} (out of {1}) of hyperband'\n                      .format(round_index + 1, s_max + 1))\n\n            for i in range((s + 1) - self.skip_last):\n\n                n_configs = np.floor(n / np.power(self.eta, i))  # n_i\n                n_iterations = int(r * np.power(self.eta, i))  # r_i\n                n_to_keep = int(np.floor(n_configs / self.eta))\n\n                if self.verbose > 0:\n                    msg = ('Starting successive halving iteration {0} out of'\n                           ' {1}. Fitting {2} configurations, with'\n                           ' resource_param {3} set to {4}')\n\n                    if n_to_keep > 0:\n                        msg += ', and keeping the best {5} configurations.'\n\n                    msg = msg.format(i + 1, s + 1, len(configurations),\n                                     self.resource_param, n_iterations,\n                                     n_to_keep)\n                    print(msg)\n\n                # Set the cost parameter for every configuration\n                parameters = copy.deepcopy(configurations)\n                for configuration in parameters:\n                    configuration[self.resource_param] = n_iterations\n\n                results = evaluate_candidates(parameters)\n\n                if n_to_keep > 0:\n                    top_configurations = [x for _, x in sorted(zip(results['rank_test_%s' % refit_metric],\n                                                                   results['params']),\n                                                               key=lambda x: x[0])]\n\n                    configurations = top_configurations[:n_to_keep]\n\n            if self.skip_last > 0:\n                print('Skipping the last {0} successive halving iterations'\n                      .format(self.skip_last))\n\n    def fit(self, X, y=None, groups=None, **fit_params):\n        \"\"\"Run fit with all sets of parameters.\n\n        Parameters\n        ----------\n        X : array-like, shape = [n_samples, n_features]\n            Training vector, where n_samples is the number of samples and\n            n_features is the number of features.\n\n        y : array-like, shape = [n_samples] or [n_samples, n_output], optional\n            Target relative to X for classification or regression;\n            None for unsupervised learning.\n\n        groups : array-like, with shape (n_samples,), optional\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\nt\n        **fit_params : dict of string -> object\n            Parameters passed to the ``fit`` method of the estimator\n        \"\"\"\n        super().fit(X=X, y=y, groups=groups, **fit_params)\n\n        s_max = int(np.floor(np.log(self.max_iter / self.min_iter) / np.log(self.eta)))\n        B = (s_max + 1) * self.max_iter\n\n        brackets = []\n        for round_index, s in enumerate(reversed(range(s_max + 1))):\n            n = int(np.ceil(int(B / self.max_iter / (s + 1)) * np.power(self.eta, s)))\n            n_configs = int(sum([np.floor(n / np.power(self.eta, i))\n                                 for i in range((s + 1) - self.skip_last)]))\n            bracket = (round_index + 1) * np.ones(n_configs)\n            brackets.append(bracket)\n\n        self.cv_results_['hyperband_bracket'] = np.hstack(brackets)\n\n        return self\n\n    def _validate_input(self):\n        if not isinstance(self.min_iter, int) or self.min_iter <= 0:\n            raise ValueError('min_iter should be a positive integer, got %s' %\n                             self.min_iter)\n\n        if not isinstance(self.max_iter, int) or self.max_iter <= 0:\n            raise ValueError('max_iter should be a positive integer, got %s' %\n                             self.max_iter)\n\n        if self.max_iter < self.min_iter:\n            raise ValueError('max_iter should be bigger than min_iter, got'\n                             'max_iter=%d and min_iter=%d' % (self.max_iter,\n                                                              self.min_iter))\n\n        if not isinstance(self.skip_last, int) or self.skip_last < 0:\n            raise ValueError('skip_last should be an integer, got %s' %\n                             self.skip_last)\n\n        if not isinstance(self.eta, int) or not self.eta > 1:\n            raise ValueError('eta should be a positive integer, got %s' %\n                             self.eta)\n\n        if self.resource_param not in self.estimator.get_params().keys():\n            raise ValueError('resource_param is set to %s, but base_estimator %s '\n                             'does not have a parameter with that name' %\n                             (self.resource_param,\n                              self.estimator.__class__.__name__))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:19:06.115919Z","iopub.execute_input":"2023-07-07T22:19:06.116287Z","iopub.status.idle":"2023-07-07T22:19:06.146097Z","shell.execute_reply.started":"2023-07-07T22:19:06.116257Z","shell.execute_reply":"2023-07-07T22:19:06.145087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_grid = {'gamma': [0,6.4,25.6],\n              'learning_rate': [0.03, 0.3, 1],\n              'max_depth': [3,6,12],\n              'n_estimators': [50,100,150],\n              'reg_alpha': [0,6.4,25.6],\n              'reg_lambda': [0,6.4,25.6]}\n\nsearch0 = HyperbandSearchCV(estimator=clf_xgb,\n                            param_distributions = param_grid,\n                            resource_param='n_estimators',\n                            scoring='roc_auc',\n                           return_train_score=True,\n                           verbose=1,\n                           cv=3)\nsearch0.fit(X_train,y_train)\nprint(search0.best_params_)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T07:09:50.877672Z","iopub.execute_input":"2023-06-16T07:09:50.878082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\n\nimport numpy as np\nfrom scipy.stats import rankdata\n\nfrom sklearn.utils import check_random_state\nfrom sklearn.model_selection._search import BaseSearchCV, ParameterSampler\n\n\n__all__ = ['HyperbandSearchCV']\n\n\nclass HyperbandSearchCV2(GridSearchCV):\n    def __init__(self, estimator, param_distributions,\n                 resource_param='n_estimators', eta=3, min_iter=1,\n                 max_iter=81, skip_last=0, scoring=None, n_jobs=1,\n                 refit=True, cv=None,\n                 verbose=0, pre_dispatch='2*n_jobs', random_state=None,\n                 error_score='raise', return_train_score=False):\n        self.param_distributions = param_distributions\n        self.resource_param = resource_param\n        self.eta = eta\n        self.min_iter = min_iter\n        self.max_iter = max_iter\n        self.skip_last = skip_last\n        self.random_state = random_state\n\n        super(HyperbandSearchCV2, self).__init__(\n            estimator=estimator,param_grid=param_distributions, scoring=scoring, n_jobs=n_jobs,\n            refit=refit, cv=cv, verbose=verbose,\n            pre_dispatch=pre_dispatch, error_score=error_score,\n            return_train_score=return_train_score)\n\n    def _run_search(self, evaluate_candidates):\n        self._validate_input()\n\n        s_max = int(np.floor(np.log(self.max_iter / self.min_iter) / np.log(self.eta)))\n        B = (s_max + 1) * self.max_iter\n\n        refit_metric = 'score'\n        random_state = check_random_state(self.random_state)\n\n        if self.skip_last > s_max:\n            raise ValueError('skip_last is higher than the total number of rounds')\n\n        for round_index, s in enumerate(reversed(range(s_max + 1))):\n            n = int(np.ceil(int(B / self.max_iter / (s + 1)) * np.power(self.eta, s)))\n\n            # initial number of iterations per config\n            r = self.max_iter / np.power(self.eta, s)\n            configurations = list(ParameterSampler(param_distributions=self.param_distributions,\n                                                   n_iter=n,\n                                                   random_state=random_state))\n\n            if self.verbose > 0:\n                print('Starting bracket {0} (out of {1}) of hyperband'\n                      .format(round_index + 1, s_max + 1))\n\n            for i in range((s + 1) - self.skip_last):\n\n                n_configs = np.floor(n / np.power(self.eta, i))  # n_i\n                n_iterations = int(r * np.power(self.eta, i))  # r_i\n                n_to_keep = int(np.floor(n_configs / self.eta))\n\n                if self.verbose > 0:\n                    msg = ('Starting successive halving iteration {0} out of'\n                           ' {1}. Fitting {2} configurations, with'\n                           ' resource_param {3} set to {4}')\n\n                    if n_to_keep > 0:\n                        msg += ', and keeping the best {5} configurations.'\n\n                    msg = msg.format(i + 1, s + 1, len(configurations),\n                                     self.resource_param, n_iterations,\n                                     n_to_keep)\n                    print(msg)\n\n                # Set the cost parameter for every configuration\n                parameters = copy.deepcopy(configurations)\n                for configuration in parameters:\n                    configuration[self.resource_param] = n_iterations\n\n                results = evaluate_candidates(parameters)\n\n                if n_to_keep > 0:\n                    top_configurations = [x for _, x in sorted(zip(results['rank_test_%s' % refit_metric],\n                                                                   results['params']),\n                                                               key=lambda x: x[0])]\n\n                    configurations = top_configurations[:n_to_keep]\n\n            if self.skip_last > 0:\n                print('Skipping the last {0} successive halving iterations'\n                      .format(self.skip_last))\n\n    def fit(self, X, y=None, groups=None, **fit_params):\n        \"\"\"Run fit with all sets of parameters.\n\n        Parameters\n        ----------\n        X : array-like, shape = [n_samples, n_features]\n            Training vector, where n_samples is the number of samples and\n            n_features is the number of features.\n\n        y : array-like, shape = [n_samples] or [n_samples, n_output], optional\n            Target relative to X for classification or regression;\n            None for unsupervised learning.\n\n        groups : array-like, with shape (n_samples,), optional\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\nt\n        **fit_params : dict of string -> object\n            Parameters passed to the ``fit`` method of the estimator\n        \"\"\"\n        print(f\"{y}\")\n        super().fit(X=X, y=y, groups=groups, **fit_params)\n\n        s_max = int(np.floor(np.log(self.max_iter / self.min_iter) / np.log(self.eta)))\n        B = (s_max + 1) * self.max_iter\n\n        brackets = []\n        for round_index, s in enumerate(reversed(range(s_max + 1))):\n            n = int(np.ceil(int(B / self.max_iter / (s + 1)) * np.power(self.eta, s)))\n            n_configs = int(sum([np.floor(n / np.power(self.eta, i))\n                                 for i in range((s + 1) - self.skip_last)]))\n            bracket = (round_index + 1) * np.ones(n_configs)\n            brackets.append(bracket)\n\n        self.cv_results_['hyperband_bracket'] = np.hstack(brackets)\n\n        return self\n\n    def _validate_input(self):\n        if not isinstance(self.min_iter, int) or self.min_iter <= 0:\n            raise ValueError('min_iter should be a positive integer, got %s' %\n                             self.min_iter)\n\n        if not isinstance(self.max_iter, int) or self.max_iter <= 0:\n            raise ValueError('max_iter should be a positive integer, got %s' %\n                             self.max_iter)\n\n        if self.max_iter < self.min_iter:\n            raise ValueError('max_iter should be bigger than min_iter, got'\n                             'max_iter=%d and min_iter=%d' % (self.max_iter,\n                                                              self.min_iter))\n\n        if not isinstance(self.skip_last, int) or self.skip_last < 0:\n            raise ValueError('skip_last should be an integer, got %s' %\n                             self.skip_last)\n\n        if not isinstance(self.eta, int) or not self.eta > 1:\n            raise ValueError('eta should be a positive integer, got %s' %\n                             self.eta)\n\n        if self.resource_param not in self.estimator.get_params().keys():\n            raise ValueError('resource_param is set to %s, but base_estimator %s '\n                             'does not have a parameter with that name' %\n                             (self.resource_param,\n                              self.estimator.__class__.__name__))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-07T22:19:22.433198Z","iopub.execute_input":"2023-07-07T22:19:22.433682Z","iopub.status.idle":"2023-07-07T22:19:22.46288Z","shell.execute_reply.started":"2023-07-07T22:19:22.433556Z","shell.execute_reply":"2023-07-07T22:19:22.461929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_grid = {'gamma': [0,6.4,25.6],\n              'learning_rate': [0.03, 0.3, 1],\n              'max_depth': [3,6,12],\n              'n_estimators': [50,100,150],\n              'reg_alpha': [0,6.4,25.6],\n              'reg_lambda': [0,6.4,25.6]}\n\ngridsearch0 = HyperbandSearchCV2(estimator=clf_xgb,\n                            param_distributions = param_grid,\n                            resource_param='n_estimators',\n                            scoring='accuracy',\n                           return_train_score=True,\n                           verbose=1)\ngridsearch0.fit(X_train,y_train)\nprint(gridsearch0.best_params_)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T05:02:51.47409Z","iopub.execute_input":"2023-06-15T05:02:51.474936Z","iopub.status.idle":"2023-06-15T06:36:04.471733Z","shell.execute_reply.started":"2023-06-15T05:02:51.474902Z","shell.execute_reply":"2023-06-15T06:36:04.470071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gridsearch0.best_params_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gridsearch0.best_params_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df = pd.DataFrame(gridsearch0.cv_results_)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T06:47:49.415733Z","iopub.execute_input":"2023-06-15T06:47:49.416109Z","iopub.status.idle":"2023-06-15T06:47:49.434241Z","shell.execute_reply.started":"2023-06-15T06:47:49.416081Z","shell.execute_reply":"2023-06-15T06:47:49.433257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df","metadata":{"execution":{"iopub.status.busy":"2023-06-15T06:47:56.765251Z","iopub.execute_input":"2023-06-15T06:47:56.765618Z","iopub.status.idle":"2023-06-15T06:47:56.832326Z","shell.execute_reply.started":"2023-06-15T06:47:56.765589Z","shell.execute_reply":"2023-06-15T06:47:56.831126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df[(results_df['param_reg_lambda'] == 6.4) & (results_df['param_reg_alpha'] == 25.6) & (results_df['param_n_estimators'] == 1)]","metadata":{"execution":{"iopub.status.busy":"2023-06-15T07:00:55.949508Z","iopub.execute_input":"2023-06-15T07:00:55.94986Z","iopub.status.idle":"2023-06-15T07:00:55.987939Z","shell.execute_reply.started":"2023-06-15T07:00:55.949832Z","shell.execute_reply":"2023-06-15T07:00:55.986864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_predictions = gridsearch0.predict(X_train,y_train)\ntest_predicition = gridsearch0.predict(X_test,y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}