nnetsauce

 21class AdaBoostClassifier(Boosting, ClassifierMixin):
 22    """AdaBoost Classification (SAMME) model class derived from class Boosting
 23
 24    Parameters:
 25
 26        obj: object
 27            any object containing a method fit (obj.fit()) and a method predict
 28            (obj.predict())
 29
 30        n_estimators: int
 31            number of boosting iterations
 32
 33        learning_rate: float
 34            learning rate of the boosting procedure
 35
 36        n_hidden_features: int
 37            number of nodes in the hidden layer
 38
 39        reg_lambda: float
 40            regularization parameter for weights
 41
 42        reg_alpha: float
 43            controls compromize between l1 and l2 norm of weights
 44
 45        activation_name: str
 46            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 47
 48        a: float
 49            hyperparameter for 'prelu' or 'elu' activation function
 50
 51        nodes_sim: str
 52            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 53            'uniform'
 54
 55        bias: boolean
 56            indicates if the hidden layer contains a bias term (True) or not
 57            (False)
 58
 59        dropout: float
 60            regularization parameter; (random) percentage of nodes dropped out
 61            of the training
 62
 63        direct_link: boolean
 64            indicates if the original predictors are included (True) in model's
 65            fitting or not (False)
 66
 67        n_clusters: int
 68            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 69                no clustering)
 70
 71        cluster_encode: bool
 72            defines how the variable containing clusters is treated (default is one-hot)
 73            if `False`, then labels are used, without one-hot encoding
 74
 75        type_clust: str
 76            type of clustering method: currently k-means ('kmeans') or Gaussian
 77            Mixture Model ('gmm')
 78
 79        type_scaling: a tuple of 3 strings
 80            scaling methods for inputs, hidden layer, and clustering respectively
 81            (and when relevant).
 82            Currently available: standardization ('std') or MinMax scaling ('minmax')
 83
 84        col_sample: float
 85            percentage of covariates randomly chosen for training
 86
 87        row_sample: float
 88            percentage of rows chosen for training, by stratified bootstrapping
 89
 90        seed: int
 91            reproducibility seed for nodes_sim=='uniform'
 92
 93        verbose: int
 94            0 for no output, 1 for a progress bar (default is 1)
 95
 96        method: str
 97            type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
 98
 99        backend: str
100            "cpu" or "gpu" or "tpu"
101
102    Attributes:
103
104        alpha_: list
105            AdaBoost coefficients alpha_m
106
107        base_learners_: dict
108            a dictionary containing the base learners
109
110    Examples:
111
112    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py)
113
114    ```python
115    import nnetsauce as ns
116    import numpy as np
117    from sklearn.datasets import load_breast_cancer
118    from sklearn.linear_model import LogisticRegression
119    from sklearn.model_selection import train_test_split
120    from sklearn import metrics
121    from time import time
122
123    breast_cancer = load_breast_cancer()
124    Z = breast_cancer.data
125    t = breast_cancer.target
126    np.random.seed(123)
127    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
128
129    # SAMME.R
130    clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
131                            random_state=123)
132    fit_obj = ns.AdaBoostClassifier(clf,
133                                    n_hidden_features=int(11.22338867),
134                                    direct_link=True,
135                                    n_estimators=250, learning_rate=0.01126343,
136                                    col_sample=0.72684326, row_sample=0.86429443,
137                                    dropout=0.63078613, n_clusters=2,
138                                    type_clust="gmm",
139                                    verbose=1, seed = 123,
140                                    method="SAMME.R")
141
142    start = time()
143    fit_obj.fit(X_train, y_train)
144    print(f"Elapsed {time() - start}")
145
146    start = time()
147    print(fit_obj.score(X_test, y_test))
148    print(f"Elapsed {time() - start}")
149
150    preds = fit_obj.predict(X_test)
151
152    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
153    print(metrics.classification_report(preds, y_test))
154
155    ```
156
157    """
158
159    # construct the object -----
160
161    def __init__(
162        self,
163        obj,
164        n_estimators=10,
165        learning_rate=0.1,
166        n_hidden_features=1,
167        reg_lambda=0,
168        reg_alpha=0.5,
169        activation_name="relu",
170        a=0.01,
171        nodes_sim="sobol",
172        bias=True,
173        dropout=0,
174        direct_link=False,
175        n_clusters=2,
176        cluster_encode=True,
177        type_clust="kmeans",
178        type_scaling=("std", "std", "std"),
179        col_sample=1,
180        row_sample=1,
181        seed=123,
182        verbose=1,
183        method="SAMME",
184        backend="cpu",
185    ):
186        self.type_fit = "classification"
187        self.verbose = verbose
188        self.method = method
189        self.reg_lambda = reg_lambda
190        self.reg_alpha = reg_alpha
191
192        super().__init__(
193            obj=obj,
194            n_estimators=n_estimators,
195            learning_rate=learning_rate,
196            n_hidden_features=n_hidden_features,
197            activation_name=activation_name,
198            a=a,
199            nodes_sim=nodes_sim,
200            bias=bias,
201            dropout=dropout,
202            direct_link=direct_link,
203            n_clusters=n_clusters,
204            cluster_encode=cluster_encode,
205            type_clust=type_clust,
206            type_scaling=type_scaling,
207            col_sample=col_sample,
208            row_sample=row_sample,
209            seed=seed,
210            backend=backend,
211        )
212
213        self.alpha_ = []
214        self.base_learners_ = dict.fromkeys(range(n_estimators))
215
216    def fit(self, X, y, sample_weight=None, **kwargs):
217        """Fit Boosting model to training data (X, y).
218
219        Parameters:
220
221            X: {array-like}, shape = [n_samples, n_features]
222                Training vectors, where n_samples is the number
223                of samples and n_features is the number of features.
224
225            y: array-like, shape = [n_samples]
226                Target values.
227
228            **kwargs: additional parameters to be passed to
229                    self.cook_training_set or self.obj.fit
230
231        Returns:
232
233             self: object
234        """
235
236        assert mx.is_factor(y), "y must contain only integers"
237
238        assert self.method in (
239            "SAMME",
240            "SAMME.R",
241        ), "`method` must be either 'SAMME' or 'SAMME.R'"
242
243        assert (self.reg_lambda <= 1) & (
244            self.reg_lambda >= 0
245        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
246
247        assert (self.reg_alpha <= 1) & (
248            self.reg_alpha >= 0
249        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
250
251        # training
252        n, p = X.shape
253        self.n_classes = len(np.unique(y))
254        self.classes_ = np.unique(y)  # for compatibility with sklearn
255        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
256
257        if sample_weight is None:
258            w_m = np.repeat(1.0 / n, n)
259        else:
260            w_m = np.asarray(sample_weight)
261
262        base_learner = CustomClassifier(
263            self.obj,
264            n_hidden_features=self.n_hidden_features,
265            activation_name=self.activation_name,
266            a=self.a,
267            nodes_sim=self.nodes_sim,
268            bias=self.bias,
269            dropout=self.dropout,
270            direct_link=self.direct_link,
271            n_clusters=self.n_clusters,
272            type_clust=self.type_clust,
273            type_scaling=self.type_scaling,
274            col_sample=self.col_sample,
275            row_sample=self.row_sample,
276            seed=self.seed,
277        )
278
279        if self.verbose == 1:
280            pbar = Progbar(self.n_estimators)
281
282        if self.method == "SAMME":
283            err_m = 1e6
284            err_bound = 1 - 1 / self.n_classes
285            self.alpha_.append(1.0)
286            x_range_n = range(n)
287
288            for m in range(self.n_estimators):
289                preds = base_learner.fit(
290                    X, y, sample_weight=w_m.ravel(), **kwargs
291                ).predict(X)
292
293                self.base_learners_.update(
294                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
295                )
296
297                cond = [y[i] != preds[i] for i in x_range_n]
298
299                err_m = max(
300                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
301                    2.220446049250313e-16,
302                )  # sum(w_m) == 1
303
304                if self.reg_lambda > 0:
305                    err_m += self.reg_lambda * (
306                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
307                        + self.reg_alpha * sum([abs(x) for x in w_m])
308                    )
309
310                err_m = min(err_m, err_bound)
311
312                alpha_m = self.learning_rate * log(
313                    (self.n_classes - 1) * (1 - err_m) / err_m
314                )
315
316                self.alpha_.append(alpha_m)
317
318                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
319
320                sum_w_m = sum(w_m_temp)
321
322                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
323
324                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
325
326                if self.verbose == 1:
327                    pbar.update(m)
328
329            if self.verbose == 1:
330                pbar.update(self.n_estimators)
331
332            self.n_estimators = len(self.base_learners_)
333            self.classes_ = np.unique(y)
334
335            return self
336
337        if self.method == "SAMME.R":
338            Y = mo.one_hot_encode2(y, self.n_classes)
339
340            if sample_weight is None:
341                w_m = np.repeat(1.0 / n, n)  # (N, 1)
342
343            else:
344                w_m = np.asarray(sample_weight)
345
346            for m in range(self.n_estimators):
347                probs = base_learner.fit(
348                    X, y, sample_weight=w_m.ravel(), **kwargs
349                ).predict_proba(X)
350
351                np.clip(
352                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
353                )
354
355                self.base_learners_.update(
356                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
357                )
358
359                w_m *= np.exp(
360                    -1.0
361                    * self.learning_rate
362                    * (1.0 - 1.0 / self.n_classes)
363                    * xlogy(Y, probs).sum(axis=1)
364                )
365
366                w_m /= np.sum(w_m)
367
368                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
369
370                if self.verbose == 1:
371                    pbar.update(m)
372
373            if self.verbose == 1:
374                pbar.update(self.n_estimators)
375
376            self.n_estimators = len(self.base_learners_)
377            self.classes_ = np.unique(y)
378
379            return self
380
381    def predict(self, X, **kwargs):
382        """Predict test data X.
383
384        Parameters:
385
386            X: {array-like}, shape = [n_samples, n_features]
387                Training vectors, where n_samples is the number
388                of samples and n_features is the number of features.
389
390            **kwargs: additional parameters to be passed to
391                  self.cook_test_set
392
393        Returns:
394
395            model predictions: {array-like}
396        """
397        return self.predict_proba(X, **kwargs).argmax(axis=1)
398
399    def predict_proba(self, X, **kwargs):
400        """Predict probabilities for test data X.
401
402        Parameters:
403
404            X: {array-like}, shape = [n_samples, n_features]
405                Training vectors, where n_samples is the number
406                of samples and n_features is the number of features.
407
408            **kwargs: additional parameters to be passed to
409                  self.cook_test_set
410
411        Returns:
412
413            probability estimates for test data: {array-like}
414
415        """
416
417        n_iter = len(self.base_learners_)
418
419        if self.method == "SAMME":
420            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
421
422            # if self.verbose == 1:
423            #    pbar = Progbar(n_iter)
424
425            for idx, base_learner in self.base_learners_.items():
426                preds = base_learner.predict(X, **kwargs)
427
428                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
429                    preds, self.n_classes
430                )
431
432                # if self.verbose == 1:
433                #    pbar.update(idx)
434
435            # if self.verbose == 1:
436            #    pbar.update(n_iter)
437
438            expit_ensemble_learner = expit(ensemble_learner)
439
440            sum_ensemble = expit_ensemble_learner.sum(axis=1)
441
442            return expit_ensemble_learner / sum_ensemble[:, None]
443
444        # if self.method == "SAMME.R":
445        ensemble_learner = 0
446
447        # if self.verbose == 1:
448        #    pbar = Progbar(n_iter)
449
450        for idx, base_learner in self.base_learners_.items():
451            probs = base_learner.predict_proba(X, **kwargs)
452
453            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
454
455            log_preds_proba = np.log(probs)
456
457            ensemble_learner += (
458                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
459            )
460
461            # if self.verbose == 1:
462            #    pbar.update(idx)
463
464        ensemble_learner *= self.n_classes - 1
465
466        # if self.verbose == 1:
467        #    pbar.update(n_iter)
468
469        expit_ensemble_learner = expit(ensemble_learner)
470
471        sum_ensemble = expit_ensemble_learner.sum(axis=1)
472
473        return expit_ensemble_learner / sum_ensemble[:, None]

 34class Base(BaseEstimator):
 35    """Base model from which all the other classes inherit.
 36
 37    This class contains the most important data preprocessing/feature engineering methods.
 38
 39    Parameters:
 40
 41        n_hidden_features: int
 42            number of nodes in the hidden layer
 43
 44        activation_name: str
 45            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 46
 47        a: float
 48            hyperparameter for 'prelu' or 'elu' activation function
 49
 50        nodes_sim: str
 51            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 52            'uniform'
 53
 54        bias: boolean
 55            indicates if the hidden layer contains a bias term (True) or
 56            not (False)
 57
 58        dropout: float
 59            regularization parameter; (random) percentage of nodes dropped out
 60            of the training
 61
 62        direct_link: boolean
 63            indicates if the original features are included (True) in model's
 64            fitting or not (False)
 65
 66        n_clusters: int
 67            number of clusters for type_clust='kmeans' or type_clust='gmm'
 68            clustering (could be 0: no clustering)
 69
 70        cluster_encode: bool
 71            defines how the variable containing clusters is treated (default is one-hot);
 72            if `False`, then labels are used, without one-hot encoding
 73
 74        type_clust: str
 75            type of clustering method: currently k-means ('kmeans') or Gaussian
 76            Mixture Model ('gmm')
 77
 78        type_scaling: a tuple of 3 strings
 79            scaling methods for inputs, hidden layer, and clustering respectively
 80            (and when relevant).
 81            Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust') or  max absolute scaling ('maxabs')
 82
 83        col_sample: float
 84            percentage of features randomly chosen for training
 85
 86        row_sample: float
 87            percentage of rows chosen for training, by stratified bootstrapping
 88
 89        seed: int
 90            reproducibility seed for nodes_sim=='uniform', clustering and dropout
 91
 92        backend: str
 93            "cpu" or "gpu" or "tpu"
 94
 95    """
 96
 97    # construct the object -----
 98
 99    def __init__(
100        self,
101        n_hidden_features=5,
102        activation_name="relu",
103        a=0.01,
104        nodes_sim="sobol",
105        bias=True,
106        dropout=0,
107        direct_link=True,
108        n_clusters=2,
109        cluster_encode=True,
110        type_clust="kmeans",
111        type_scaling=("std", "std", "std"),
112        col_sample=1,
113        row_sample=1,
114        seed=123,
115        backend="cpu",
116    ):
117        # input checks -----
118
119        sys_platform = platform.system()
120
121        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
122            warnings.warn(
123                "No GPU/TPU computing on Windows yet, backend set to 'cpu'"
124            )
125            backend = "cpu"
126
127        assert activation_name in (
128            "relu",
129            "tanh",
130            "sigmoid",
131            "prelu",
132            "elu",
133        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
134
135        assert nodes_sim in (
136            "sobol",
137            "hammersley",
138            "uniform",
139            "halton",
140        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
141
142        assert type_clust in (
143            "kmeans",
144            "gmm",
145        ), "'type_clust' must be in ('kmeans', 'gmm')"
146
147        assert (len(type_scaling) == 3) & all(
148            type_scaling[i] in ("minmax", "std", "robust", "maxabs")
149            for i in range(len(type_scaling))
150        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std'), robust scaling ('robust') and max absolute ('maxabs')"
151
152        assert (col_sample >= 0) & (
153            col_sample <= 1
154        ), "'col_sample' must be comprised between 0 and 1 (both included)"
155
156        assert backend in (
157            "cpu",
158            "gpu",
159            "tpu",
160        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
161
162        self.n_hidden_features = n_hidden_features
163        self.activation_name = activation_name
164        self.a = a
165        self.nodes_sim = nodes_sim
166        self.bias = bias
167        self.seed = seed
168        self.backend = backend
169        self.dropout = dropout
170        self.direct_link = direct_link
171        self.cluster_encode = cluster_encode
172        self.type_clust = type_clust
173        self.type_scaling = type_scaling
174        self.col_sample = col_sample
175        self.row_sample = row_sample
176        self.n_clusters = n_clusters
177        if isinstance(self, RegressorMixin):
178            self.type_fit = "regression"
179        elif isinstance(self, ClassifierMixin):
180            self.type_fit = "classification"
181        self.subsampler_ = None
182        self.index_col_ = None
183        self.index_row_ = True
184        self.clustering_obj_ = None
185        self.clustering_scaler_ = None
186        self.nn_scaler_ = None
187        self.scaler_ = None
188        self.encoder_ = None
189        self.W_ = None
190        self.X_ = None
191        self.y_ = None
192        self.y_mean_ = None
193        self.beta_ = None
194
195        # activation function -----
196        if sys_platform in ("Linux", "Darwin"):
197            activation_options = {
198                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
199                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
200                "sigmoid": (
201                    ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid
202                ),
203                "prelu": partial(ac.prelu, a=a),
204                "elu": (
205                    partial(ac.elu, a=a)
206                    if (self.backend == "cpu")
207                    else partial(jnn.elu, a=a)
208                ),
209            }
210        else:  # on Windows currently, no JAX
211            activation_options = {
212                "relu": (
213                    ac.relu if (self.backend == "cpu") else NotImplementedError
214                ),
215                "tanh": (
216                    np.tanh if (self.backend == "cpu") else NotImplementedError
217                ),
218                "sigmoid": (
219                    ac.sigmoid
220                    if (self.backend == "cpu")
221                    else NotImplementedError
222                ),
223                "prelu": partial(ac.prelu, a=a),
224                "elu": (
225                    partial(ac.elu, a=a)
226                    if (self.backend == "cpu")
227                    else NotImplementedError
228                ),
229            }
230        self.activation_func = activation_options[activation_name]
231
232    # "preprocessing" methods to be inherited -----
233
234    def encode_clusters(self, X=None, predict=False, **kwargs):  #
235        """Create new covariates with kmeans or GMM clustering
236
237        Parameters:
238
239            X: {array-like}, shape = [n_samples, n_features]
240                Training vectors, where n_samples is the number
241                of samples and n_features is the number of features.
242
243            predict: boolean
244                is False on training set and True on test set
245
246            **kwargs:
247                additional parameters to be passed to the
248                clustering method
249
250        Returns:
251
252            Clusters' matrix, one-hot encoded: {array-like}
253
254        """
255
256        np.random.seed(self.seed)
257
258        if X is None:
259            X = self.X_
260
261        if isinstance(X, pd.DataFrame):
262            X = copy.deepcopy(X.values.astype(float))
263
264        if predict is False:  # encode training set
265            # scale input data before clustering
266            self.clustering_scaler_, scaled_X = mo.scale_covariates(
267                X, choice=self.type_scaling[2]
268            )
269
270            self.clustering_obj_, X_clustered = mo.cluster_covariates(
271                scaled_X,
272                self.n_clusters,
273                self.seed,
274                type_clust=self.type_clust,
275                **kwargs
276            )
277
278            if self.cluster_encode == True:
279                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
280                    np.float16
281                )
282
283            return X_clustered.astype(np.float16)
284
285        # if predict == True, encode test set
286        X_clustered = self.clustering_obj_.predict(
287            self.clustering_scaler_.transform(X)
288        )
289
290        if self.cluster_encode == True:
291            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
292                np.float16
293            )
294
295        return X_clustered.astype(np.float16)
296
297    def create_layer(self, scaled_X, W=None):
298        """Create hidden layer.
299
300        Parameters:
301
302            scaled_X: {array-like}, shape = [n_samples, n_features]
303                Training vectors, where n_samples is the number
304                of samples and n_features is the number of features
305
306            W: {array-like}, shape = [n_features, hidden_features]
307                if provided, constructs the hidden layer with W; otherwise computed internally
308
309        Returns:
310
311            Hidden layer matrix: {array-like}
312
313        """
314
315        n_features = scaled_X.shape[1]
316
317        # hash_sim = {
318        #         "sobol": generate_sobol,
319        #         "hammersley": generate_hammersley,
320        #         "uniform": generate_uniform,
321        #         "halton": generate_halton
322        #     }
323
324        if self.bias is False:  # no bias term in the hidden layer
325            if W is None:
326                if self.nodes_sim == "sobol":
327                    self.W_ = generate_sobol(
328                        n_dims=n_features,
329                        n_points=self.n_hidden_features,
330                        seed=self.seed,
331                    )
332                elif self.nodes_sim == "hammersley":
333                    self.W_ = generate_hammersley(
334                        n_dims=n_features,
335                        n_points=self.n_hidden_features,
336                        seed=self.seed,
337                    )
338                elif self.nodes_sim == "uniform":
339                    self.W_ = generate_uniform(
340                        n_dims=n_features,
341                        n_points=self.n_hidden_features,
342                        seed=self.seed,
343                    )
344                else:
345                    self.W_ = generate_halton(
346                        n_dims=n_features,
347                        n_points=self.n_hidden_features,
348                        seed=self.seed,
349                    )
350
351                # self.W_ = hash_sim[self.nodes_sim](
352                #             n_dims=n_features,
353                #             n_points=self.n_hidden_features,
354                #             seed=self.seed,
355                #         )
356
357                assert (
358                    scaled_X.shape[1] == self.W_.shape[0]
359                ), "check dimensions of covariates X and matrix W"
360
361                return mo.dropout(
362                    x=self.activation_func(
363                        mo.safe_sparse_dot(
364                            a=scaled_X, b=self.W_, backend=self.backend
365                        )
366                    ),
367                    drop_prob=self.dropout,
368                    seed=self.seed,
369                )
370
371            # W is not none
372            assert (
373                scaled_X.shape[1] == W.shape[0]
374            ), "check dimensions of covariates X and matrix W"
375
376            # self.W_ = W
377            return mo.dropout(
378                x=self.activation_func(
379                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
380                ),
381                drop_prob=self.dropout,
382                seed=self.seed,
383            )
384
385        # with bias term in the hidden layer
386        if W is None:
387            n_features_1 = n_features + 1
388
389            if self.nodes_sim == "sobol":
390                self.W_ = generate_sobol(
391                    n_dims=n_features_1,
392                    n_points=self.n_hidden_features,
393                    seed=self.seed,
394                )
395            elif self.nodes_sim == "hammersley":
396                self.W_ = generate_hammersley(
397                    n_dims=n_features_1,
398                    n_points=self.n_hidden_features,
399                    seed=self.seed,
400                )
401            elif self.nodes_sim == "uniform":
402                self.W_ = generate_uniform(
403                    n_dims=n_features_1,
404                    n_points=self.n_hidden_features,
405                    seed=self.seed,
406                )
407            else:
408                self.W_ = generate_halton(
409                    n_dims=n_features_1,
410                    n_points=self.n_hidden_features,
411                    seed=self.seed,
412                )
413
414            # self.W_ = hash_sim[self.nodes_sim](
415            #         n_dims=n_features_1,
416            #         n_points=self.n_hidden_features,
417            #         seed=self.seed,
418            #     )
419
420            return mo.dropout(
421                x=self.activation_func(
422                    mo.safe_sparse_dot(
423                        a=mo.cbind(
424                            np.ones(scaled_X.shape[0]),
425                            scaled_X,
426                            backend=self.backend,
427                        ),
428                        b=self.W_,
429                        backend=self.backend,
430                    )
431                ),
432                drop_prob=self.dropout,
433                seed=self.seed,
434            )
435
436        # W is not None
437        # self.W_ = W
438        return mo.dropout(
439            x=self.activation_func(
440                mo.safe_sparse_dot(
441                    a=mo.cbind(
442                        np.ones(scaled_X.shape[0]),
443                        scaled_X,
444                        backend=self.backend,
445                    ),
446                    b=W,
447                    backend=self.backend,
448                )
449            ),
450            drop_prob=self.dropout,
451            seed=self.seed,
452        )
453
454    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
455        """Create new hidden features for training set, with hidden layer, center the response.
456
457        Parameters:
458
459            y: array-like, shape = [n_samples]
460                Target values
461
462            X: {array-like}, shape = [n_samples, n_features]
463                Training vectors, where n_samples is the number
464                of samples and n_features is the number of features
465
466            W: {array-like}, shape = [n_features, hidden_features]
467                if provided, constructs the hidden layer via W
468
469        Returns:
470
471            (centered response, direct link + hidden layer matrix): {tuple}
472
473        """
474
475        # either X and y are stored or not
476        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
477        if self.n_hidden_features > 0:  # has a hidden layer
478            assert (
479                len(self.type_scaling) >= 2
480            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
481
482        if X is None:
483            if self.col_sample == 1:
484                input_X = self.X_
485            else:
486                n_features = self.X_.shape[1]
487                new_n_features = int(np.ceil(n_features * self.col_sample))
488                assert (
489                    new_n_features >= 1
490                ), "check class attribute 'col_sample' and the number of covariates provided for X"
491                np.random.seed(self.seed)
492                index_col = np.random.choice(
493                    range(n_features), size=new_n_features, replace=False
494                )
495                self.index_col_ = index_col
496                input_X = self.X_[:, self.index_col_]
497
498        else:  # X is not None # keep X vs self.X_
499            if isinstance(X, pd.DataFrame):
500                X = copy.deepcopy(X.values.astype(float))
501
502            if self.col_sample == 1:
503                input_X = X
504            else:
505                n_features = X.shape[1]
506                new_n_features = int(np.ceil(n_features * self.col_sample))
507                assert (
508                    new_n_features >= 1
509                ), "check class attribute 'col_sample' and the number of covariates provided for X"
510                np.random.seed(self.seed)
511                index_col = np.random.choice(
512                    range(n_features), size=new_n_features, replace=False
513                )
514                self.index_col_ = index_col
515                input_X = X[:, self.index_col_]
516
517        if (
518            self.n_clusters <= 0
519        ):  # data without any clustering: self.n_clusters is None -----
520            if self.n_hidden_features > 0:  # with hidden layer
521                self.nn_scaler_, scaled_X = mo.scale_covariates(
522                    input_X, choice=self.type_scaling[1]
523                )
524                Phi_X = (
525                    self.create_layer(scaled_X)
526                    if W is None
527                    else self.create_layer(scaled_X, W=W)
528                )
529                Z = (
530                    mo.cbind(input_X, Phi_X, backend=self.backend)
531                    if self.direct_link is True
532                    else Phi_X
533                )
534                self.scaler_, scaled_Z = mo.scale_covariates(
535                    Z, choice=self.type_scaling[0]
536                )
537            else:  # no hidden layer
538                Z = input_X
539                self.scaler_, scaled_Z = mo.scale_covariates(
540                    Z, choice=self.type_scaling[0]
541                )
542        else:  # data with clustering: self.n_clusters is not None ----- # keep
543            augmented_X = mo.cbind(
544                input_X,
545                self.encode_clusters(input_X, **kwargs),
546                backend=self.backend,
547            )
548
549            if self.n_hidden_features > 0:  # with hidden layer
550                self.nn_scaler_, scaled_X = mo.scale_covariates(
551                    augmented_X, choice=self.type_scaling[1]
552                )
553                Phi_X = (
554                    self.create_layer(scaled_X)
555                    if W is None
556                    else self.create_layer(scaled_X, W=W)
557                )
558                Z = (
559                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
560                    if self.direct_link is True
561                    else Phi_X
562                )
563                self.scaler_, scaled_Z = mo.scale_covariates(
564                    Z, choice=self.type_scaling[0]
565                )
566            else:  # no hidden layer
567                Z = augmented_X
568                self.scaler_, scaled_Z = mo.scale_covariates(
569                    Z, choice=self.type_scaling[0]
570                )
571
572        # Returning model inputs -----
573        if mx.is_factor(y) is False:  # regression
574            # center y
575            if y is None:
576                self.y_mean_, centered_y = mo.center_response(self.y_)
577            else:
578                self.y_mean_, centered_y = mo.center_response(y)
579
580            # y is subsampled
581            if self.row_sample < 1:
582                n, p = Z.shape
583
584                self.subsampler_ = (
585                    SubSampler(
586                        y=self.y_, row_sample=self.row_sample, seed=self.seed
587                    )
588                    if y is None
589                    else SubSampler(
590                        y=y, row_sample=self.row_sample, seed=self.seed
591                    )
592                )
593
594                self.index_row_ = self.subsampler_.subsample()
595
596                n_row_sample = len(self.index_row_)
597                # regression
598                return (
599                    centered_y[self.index_row_].reshape(n_row_sample),
600                    self.scaler_.transform(
601                        Z[self.index_row_, :].reshape(n_row_sample, p)
602                    ),
603                )
604            # y is not subsampled
605            # regression
606            return (centered_y, self.scaler_.transform(Z))
607
608        # classification
609        # y is subsampled
610        if self.row_sample < 1:
611            n, p = Z.shape
612
613            self.subsampler_ = (
614                SubSampler(
615                    y=self.y_, row_sample=self.row_sample, seed=self.seed
616                )
617                if y is None
618                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
619            )
620
621            self.index_row_ = self.subsampler_.subsample()
622
623            n_row_sample = len(self.index_row_)
624            # classification
625            return (
626                y[self.index_row_].reshape(n_row_sample),
627                self.scaler_.transform(
628                    Z[self.index_row_, :].reshape(n_row_sample, p)
629                ),
630            )
631        # y is not subsampled
632        # classification
633        return (y, self.scaler_.transform(Z))
634
635    def cook_test_set(self, X, **kwargs):
636        """Transform data from test set, with hidden layer.
637
638        Parameters:
639
640            X: {array-like}, shape = [n_samples, n_features]
641                Training vectors, where n_samples is the number
642                of samples and n_features is the number of features
643
644            **kwargs: additional parameters to be passed to self.encode_cluster
645
646        Returns:
647
648            Transformed test set : {array-like}
649        """
650
651        if isinstance(X, pd.DataFrame):
652            X = copy.deepcopy(X.values.astype(float))
653
654        if (
655            self.n_clusters == 0
656        ):  # data without clustering: self.n_clusters is None -----
657            if self.n_hidden_features > 0:
658                # if hidden layer
659                scaled_X = (
660                    self.nn_scaler_.transform(X)
661                    if (self.col_sample == 1)
662                    else self.nn_scaler_.transform(X[:, self.index_col_])
663                )
664                Phi_X = self.create_layer(scaled_X, self.W_)
665                if self.direct_link == True:
666                    return self.scaler_.transform(
667                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
668                    )
669                # when self.direct_link == False
670                return self.scaler_.transform(Phi_X)
671            # if no hidden layer # self.n_hidden_features == 0
672            return self.scaler_.transform(X)
673
674        # data with clustering: self.n_clusters > 0 -----
675        if self.col_sample == 1:
676            predicted_clusters = self.encode_clusters(
677                X=X, predict=True, **kwargs
678            )
679            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
680        else:
681            predicted_clusters = self.encode_clusters(
682                X=X[:, self.index_col_], predict=True, **kwargs
683            )
684            augmented_X = mo.cbind(
685                X[:, self.index_col_], predicted_clusters, backend=self.backend
686            )
687
688        if self.n_hidden_features > 0:  # if hidden layer
689            scaled_X = self.nn_scaler_.transform(augmented_X)
690            Phi_X = self.create_layer(scaled_X, self.W_)
691            if self.direct_link == True:
692                return self.scaler_.transform(
693                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
694                )
695            return self.scaler_.transform(Phi_X)
696
697        # if no hidden layer
698        return self.scaler_.transform(augmented_X)
699
700    def score(self, X, y, scoring=None, **kwargs):
701        """Score the model on test set features X and response y.
702
703        Parameters:
704
705            X: {array-like}, shape = [n_samples, n_features]
706                Training vectors, where n_samples is the number
707                of samples and n_features is the number of features
708
709            y: array-like, shape = [n_samples]
710                Target values
711
712            scoring: str
713                must be in ('explained_variance', 'neg_mean_absolute_error',
714                            'neg_mean_squared_error', 'neg_mean_squared_log_error',
715                            'neg_median_absolute_error', 'r2')
716
717            **kwargs: additional parameters to be passed to scoring functions
718
719        Returns:
720
721        model scores: {array-like}
722
723        """
724
725        preds = self.predict(X)
726
727        if self.type_fit == "classification":
728
729            if scoring is None:
730                scoring = "accuracy"
731
732            # check inputs
733            assert scoring in (
734                "accuracy",
735                "average_precision",
736                "brier_score_loss",
737                "f1",
738                "f1_micro",
739                "f1_macro",
740                "f1_weighted",
741                "f1_samples",
742                "neg_log_loss",
743                "precision",
744                "recall",
745                "roc_auc",
746            ), "'scoring' should be in ('accuracy', 'average_precision', \
747                            'brier_score_loss', 'f1', 'f1_micro', \
748                            'f1_macro', 'f1_weighted',  'f1_samples', \
749                            'neg_log_loss', 'precision', 'recall', \
750                            'roc_auc')"
751
752            scoring_options = {
753                "accuracy": skm.accuracy_score,
754                "average_precision": skm.average_precision_score,
755                "brier_score_loss": skm.brier_score_loss,
756                "f1": skm.f1_score,
757                "f1_micro": skm.f1_score,
758                "f1_macro": skm.f1_score,
759                "f1_weighted": skm.f1_score,
760                "f1_samples": skm.f1_score,
761                "neg_log_loss": skm.log_loss,
762                "precision": skm.precision_score,
763                "recall": skm.recall_score,
764                "roc_auc": skm.roc_auc_score,
765            }
766
767            return scoring_options[scoring](y, preds, **kwargs)
768
769        if self.type_fit == "regression":
770
771            if (
772                type(preds) == tuple
773            ):  # if there are std. devs in the predictions
774                preds = preds[0]
775
776            if scoring is None:
777                scoring = "neg_root_mean_squared_error"
778
779            # check inputs
780            assert scoring in (
781                "explained_variance",
782                "neg_mean_absolute_error",
783                "neg_mean_squared_error",
784                "neg_mean_squared_log_error",
785                "neg_median_absolute_error",
786                "neg_root_mean_squared_error",
787                "r2",
788            ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \
789                            'neg_mean_squared_error', 'neg_mean_squared_log_error', \
790                            'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')"
791
792            scoring_options = {
793                "neg_root_mean_squared_error": skm.root_mean_squared_error,
794                "explained_variance": skm.explained_variance_score,
795                "neg_mean_absolute_error": skm.median_absolute_error,
796                "neg_mean_squared_error": skm.mean_squared_error,
797                "neg_mean_squared_log_error": skm.mean_squared_log_error,
798                "neg_median_absolute_error": skm.median_absolute_error,
799                "r2": skm.r2_score,
800            }
801
802            return scoring_options[scoring](y, preds, **kwargs)