nnetsauce

 21class AdaBoostClassifier(Boosting, ClassifierMixin):
 22    """AdaBoost Classification (SAMME) model class derived from class Boosting
 23
 24    Parameters:
 25
 26        obj: object
 27            any object containing a method fit (obj.fit()) and a method predict
 28            (obj.predict())
 29
 30        n_estimators: int
 31            number of boosting iterations
 32
 33        learning_rate: float
 34            learning rate of the boosting procedure
 35
 36        n_hidden_features: int
 37            number of nodes in the hidden layer
 38
 39        reg_lambda: float
 40            regularization parameter for weights
 41
 42        reg_alpha: float
 43            controls compromize between l1 and l2 norm of weights
 44
 45        activation_name: str
 46            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 47
 48        a: float
 49            hyperparameter for 'prelu' or 'elu' activation function
 50
 51        nodes_sim: str
 52            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 53            'uniform'
 54
 55        bias: boolean
 56            indicates if the hidden layer contains a bias term (True) or not
 57            (False)
 58
 59        dropout: float
 60            regularization parameter; (random) percentage of nodes dropped out
 61            of the training
 62
 63        direct_link: boolean
 64            indicates if the original predictors are included (True) in model's
 65            fitting or not (False)
 66
 67        n_clusters: int
 68            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 69                no clustering)
 70
 71        cluster_encode: bool
 72            defines how the variable containing clusters is treated (default is one-hot)
 73            if `False`, then labels are used, without one-hot encoding
 74
 75        type_clust: str
 76            type of clustering method: currently k-means ('kmeans') or Gaussian
 77            Mixture Model ('gmm')
 78
 79        type_scaling: a tuple of 3 strings
 80            scaling methods for inputs, hidden layer, and clustering respectively
 81            (and when relevant).
 82            Currently available: standardization ('std') or MinMax scaling ('minmax')
 83
 84        col_sample: float
 85            percentage of covariates randomly chosen for training
 86
 87        row_sample: float
 88            percentage of rows chosen for training, by stratified bootstrapping
 89
 90        seed: int
 91            reproducibility seed for nodes_sim=='uniform'
 92
 93        verbose: int
 94            0 for no output, 1 for a progress bar (default is 1)
 95
 96        method: str
 97            type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
 98
 99        backend: str
100            "cpu" or "gpu" or "tpu"
101
102    Attributes:
103
104        alpha_: list
105            AdaBoost coefficients alpha_m
106
107        base_learners_: dict
108            a dictionary containing the base learners
109
110    Examples:
111
112    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py)
113
114    ```python
115    import nnetsauce as ns
116    import numpy as np
117    from sklearn.datasets import load_breast_cancer
118    from sklearn.linear_model import LogisticRegression
119    from sklearn.model_selection import train_test_split
120    from sklearn import metrics
121    from time import time
122
123    breast_cancer = load_breast_cancer()
124    Z = breast_cancer.data
125    t = breast_cancer.target
126    np.random.seed(123)
127    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
128
129    # SAMME.R
130    clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
131                            random_state=123)
132    fit_obj = ns.AdaBoostClassifier(clf,
133                                    n_hidden_features=int(11.22338867),
134                                    direct_link=True,
135                                    n_estimators=250, learning_rate=0.01126343,
136                                    col_sample=0.72684326, row_sample=0.86429443,
137                                    dropout=0.63078613, n_clusters=2,
138                                    type_clust="gmm",
139                                    verbose=1, seed = 123,
140                                    method="SAMME.R")
141
142    start = time()
143    fit_obj.fit(X_train, y_train)
144    print(f"Elapsed {time() - start}")
145
146    start = time()
147    print(fit_obj.score(X_test, y_test))
148    print(f"Elapsed {time() - start}")
149
150    preds = fit_obj.predict(X_test)
151
152    print(metrics.classification_report(preds, y_test))
153
154    ```
155
156    """
157
158    # construct the object -----
159
160    def __init__(
161        self,
162        obj,
163        n_estimators=10,
164        learning_rate=0.1,
165        n_hidden_features=1,
166        reg_lambda=0,
167        reg_alpha=0.5,
168        activation_name="relu",
169        a=0.01,
170        nodes_sim="sobol",
171        bias=True,
172        dropout=0,
173        direct_link=False,
174        n_clusters=2,
175        cluster_encode=True,
176        type_clust="kmeans",
177        type_scaling=("std", "std", "std"),
178        col_sample=1,
179        row_sample=1,
180        seed=123,
181        verbose=1,
182        method="SAMME",
183        backend="cpu",
184    ):
185        self.type_fit = "classification"
186        self.verbose = verbose
187        self.method = method
188        self.reg_lambda = reg_lambda
189        self.reg_alpha = reg_alpha
190
191        super().__init__(
192            obj=obj,
193            n_estimators=n_estimators,
194            learning_rate=learning_rate,
195            n_hidden_features=n_hidden_features,
196            activation_name=activation_name,
197            a=a,
198            nodes_sim=nodes_sim,
199            bias=bias,
200            dropout=dropout,
201            direct_link=direct_link,
202            n_clusters=n_clusters,
203            cluster_encode=cluster_encode,
204            type_clust=type_clust,
205            type_scaling=type_scaling,
206            col_sample=col_sample,
207            row_sample=row_sample,
208            seed=seed,
209            backend=backend,
210        )
211
212        self.alpha_ = []
213        self.base_learners_ = dict.fromkeys(range(n_estimators))
214
215    def fit(self, X, y, sample_weight=None, **kwargs):
216        """Fit Boosting model to training data (X, y).
217
218        Parameters:
219
220            X: {array-like}, shape = [n_samples, n_features]
221                Training vectors, where n_samples is the number
222                of samples and n_features is the number of features.
223
224            y: array-like, shape = [n_samples]
225                Target values.
226
227            **kwargs: additional parameters to be passed to
228                    self.cook_training_set or self.obj.fit
229
230        Returns:
231
232             self: object
233        """
234
235        assert mx.is_factor(y), "y must contain only integers"
236
237        assert self.method in (
238            "SAMME",
239            "SAMME.R",
240        ), "`method` must be either 'SAMME' or 'SAMME.R'"
241
242        assert (self.reg_lambda <= 1) & (
243            self.reg_lambda >= 0
244        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
245
246        assert (self.reg_alpha <= 1) & (
247            self.reg_alpha >= 0
248        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
249
250        # training
251        n, p = X.shape
252        self.n_classes = len(np.unique(y))
253        self.classes_ = np.unique(y)  # for compatibility with sklearn
254        self.n_classes_ = len(self.classes_)  # for compatibility with sklearn
255
256        if sample_weight is None:
257            w_m = np.repeat(1.0 / n, n)
258        else:
259            w_m = np.asarray(sample_weight)
260
261        base_learner = CustomClassifier(
262            self.obj,
263            n_hidden_features=self.n_hidden_features,
264            activation_name=self.activation_name,
265            a=self.a,
266            nodes_sim=self.nodes_sim,
267            bias=self.bias,
268            dropout=self.dropout,
269            direct_link=self.direct_link,
270            n_clusters=self.n_clusters,
271            type_clust=self.type_clust,
272            type_scaling=self.type_scaling,
273            col_sample=self.col_sample,
274            row_sample=self.row_sample,
275            seed=self.seed,
276        )
277
278        if self.verbose == 1:
279            pbar = Progbar(self.n_estimators)
280
281        if self.method == "SAMME":
282            err_m = 1e6
283            err_bound = 1 - 1 / self.n_classes
284            self.alpha_.append(1.0)
285            x_range_n = range(n)
286
287            for m in range(self.n_estimators):
288                preds = base_learner.fit(
289                    X, y, sample_weight=w_m.ravel(), **kwargs
290                ).predict(X)
291
292                self.base_learners_.update({m: deepcopy(base_learner)})
293
294                cond = [y[i] != preds[i] for i in x_range_n]
295
296                err_m = max(
297                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
298                    2.220446049250313e-16,
299                )  # sum(w_m) == 1
300
301                if self.reg_lambda > 0:
302                    err_m += self.reg_lambda * (
303                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
304                        + self.reg_alpha * sum([abs(x) for x in w_m])
305                    )
306
307                err_m = min(err_m, err_bound)
308
309                alpha_m = self.learning_rate * log(
310                    (self.n_classes - 1) * (1 - err_m) / err_m
311                )
312
313                self.alpha_.append(alpha_m)
314
315                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
316
317                sum_w_m = sum(w_m_temp)
318
319                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
320
321                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
322
323                if self.verbose == 1:
324                    pbar.update(m)
325
326            if self.verbose == 1:
327                pbar.update(self.n_estimators)
328
329            self.n_estimators = len(self.base_learners_)
330            self.classes_ = np.unique(y)
331
332            return self
333
334        if self.method == "SAMME.R":
335            Y = mo.one_hot_encode2(y, self.n_classes)
336
337            if sample_weight is None:
338                w_m = np.repeat(1.0 / n, n)  # (N, 1)
339
340            else:
341                w_m = np.asarray(sample_weight)
342
343            for m in range(self.n_estimators):
344                probs = base_learner.fit(
345                    X, y, sample_weight=w_m.ravel(), **kwargs
346                ).predict_proba(X)
347
348                np.clip(
349                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
350                )
351
352                self.base_learners_.update({m: deepcopy(base_learner)})
353
354                w_m *= np.exp(
355                    -1.0
356                    * self.learning_rate
357                    * (1.0 - 1.0 / self.n_classes)
358                    * xlogy(Y, probs).sum(axis=1)
359                )
360
361                w_m /= np.sum(w_m)
362
363                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
364
365                if self.verbose == 1:
366                    pbar.update(m)
367
368            if self.verbose == 1:
369                pbar.update(self.n_estimators)
370
371            self.n_estimators = len(self.base_learners_)
372            self.classes_ = np.unique(y)
373
374            return self
375
376    def predict(self, X, **kwargs):
377        """Predict test data X.
378
379        Parameters:
380
381            X: {array-like}, shape = [n_samples, n_features]
382                Training vectors, where n_samples is the number
383                of samples and n_features is the number of features.
384
385            **kwargs: additional parameters to be passed to
386                  self.cook_test_set
387
388        Returns:
389
390            model predictions: {array-like}
391        """
392        return self.predict_proba(X, **kwargs).argmax(axis=1)
393
394    def predict_proba(self, X, **kwargs):
395        """Predict probabilities for test data X.
396
397        Parameters:
398
399            X: {array-like}, shape = [n_samples, n_features]
400                Training vectors, where n_samples is the number
401                of samples and n_features is the number of features.
402
403            **kwargs: additional parameters to be passed to
404                  self.cook_test_set
405
406        Returns:
407
408            probability estimates for test data: {array-like}
409
410        """
411
412        n_iter = len(self.base_learners_)
413
414        if self.method == "SAMME":
415            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
416
417            # if self.verbose == 1:
418            #    pbar = Progbar(n_iter)
419
420            for idx, base_learner in self.base_learners_.items():
421                preds = base_learner.predict(X, **kwargs)
422
423                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
424                    preds, self.n_classes
425                )
426
427                # if self.verbose == 1:
428                #    pbar.update(idx)
429
430            # if self.verbose == 1:
431            #    pbar.update(n_iter)
432
433            expit_ensemble_learner = expit(ensemble_learner)
434
435            sum_ensemble = expit_ensemble_learner.sum(axis=1)
436
437            return expit_ensemble_learner / sum_ensemble[:, None]
438
439        # if self.method == "SAMME.R":
440        ensemble_learner = 0
441
442        # if self.verbose == 1:
443        #    pbar = Progbar(n_iter)
444
445        for idx, base_learner in self.base_learners_.items():
446            probs = base_learner.predict_proba(X, **kwargs)
447
448            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
449
450            log_preds_proba = np.log(probs)
451
452            ensemble_learner += (
453                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
454            )
455
456            # if self.verbose == 1:
457            #    pbar.update(idx)
458
459        ensemble_learner *= self.n_classes - 1
460
461        # if self.verbose == 1:
462        #    pbar.update(n_iter)
463
464        expit_ensemble_learner = expit(ensemble_learner)
465
466        sum_ensemble = expit_ensemble_learner.sum(axis=1)
467
468        return expit_ensemble_learner / sum_ensemble[:, None]

 46class Base(BaseEstimator):
 47    """Base model from which all the other classes inherit.
 48
 49    This class contains the most important data preprocessing/feature engineering methods.
 50
 51    Parameters:
 52
 53        n_hidden_features: int
 54            number of nodes in the hidden layer
 55
 56        activation_name: str
 57            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 58
 59        a: float
 60            hyperparameter for 'prelu' or 'elu' activation function
 61
 62        nodes_sim: str
 63            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 64            'uniform'
 65
 66        bias: boolean
 67            indicates if the hidden layer contains a bias term (True) or
 68            not (False)
 69
 70        dropout: float
 71            regularization parameter; (random) percentage of nodes dropped out
 72            of the training
 73
 74        direct_link: boolean
 75            indicates if the original features are included (True) in model's
 76            fitting or not (False)
 77
 78        n_clusters: int
 79            number of clusters for type_clust='kmeans' or type_clust='gmm'
 80            clustering (could be 0: no clustering)
 81
 82        cluster_encode: bool
 83            defines how the variable containing clusters is treated (default is one-hot);
 84            if `False`, then labels are used, without one-hot encoding
 85
 86        type_clust: str
 87            type of clustering method: currently k-means ('kmeans') or Gaussian
 88            Mixture Model ('gmm')
 89
 90        type_scaling: a tuple of 3 strings
 91            scaling methods for inputs, hidden layer, and clustering respectively
 92            (and when relevant).
 93            Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust') or  max absolute scaling ('maxabs')
 94
 95        col_sample: float
 96            percentage of features randomly chosen for training
 97
 98        row_sample: float
 99            percentage of rows chosen for training, by stratified bootstrapping
100
101        seed: int
102            reproducibility seed for nodes_sim=='uniform', clustering and dropout
103
104        backend: str
105            "cpu" or "gpu" or "tpu"
106
107    """
108
109    # construct the object -----
110
111    def __init__(
112        self,
113        n_hidden_features=5,
114        activation_name="relu",
115        a=0.01,
116        nodes_sim="sobol",
117        bias=True,
118        dropout=0,
119        direct_link=True,
120        n_clusters=2,
121        cluster_encode=True,
122        type_clust="kmeans",
123        type_scaling=("std", "std", "std"),
124        col_sample=1,
125        row_sample=1,
126        seed=123,
127        backend="cpu",
128    ):
129        # input checks -----
130
131        sys_platform = platform.system()
132
133        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
134            warnings.warn(
135                "No GPU/TPU computing on Windows yet, backend set to 'cpu'"
136            )
137            backend = "cpu"
138
139        assert activation_name in (
140            "relu",
141            "tanh",
142            "sigmoid",
143            "prelu",
144            "elu",
145        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
146
147        assert nodes_sim in (
148            "sobol",
149            "hammersley",
150            "uniform",
151            "halton",
152        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
153
154        assert type_clust in (
155            "kmeans",
156            "gmm",
157        ), "'type_clust' must be in ('kmeans', 'gmm')"
158
159        assert (len(type_scaling) == 3) & all(
160            type_scaling[i] in ("minmax", "std", "robust", "maxabs")
161            for i in range(len(type_scaling))
162        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std'), robust scaling ('robust') and max absolute ('maxabs')"
163
164        assert (col_sample >= 0) & (
165            col_sample <= 1
166        ), "'col_sample' must be comprised between 0 and 1 (both included)"
167
168        assert backend in (
169            "cpu",
170            "gpu",
171            "tpu",
172        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
173
174        self.n_hidden_features = n_hidden_features
175        self.activation_name = activation_name
176        self.a = a
177        self.nodes_sim = nodes_sim
178        self.bias = bias
179        self.seed = seed
180        self.backend = backend
181        self.dropout = dropout
182        self.direct_link = direct_link
183        self.cluster_encode = cluster_encode
184        self.type_clust = type_clust
185        self.type_scaling = type_scaling
186        self.col_sample = col_sample
187        self.row_sample = row_sample
188        self.n_clusters = n_clusters
189        if isinstance(self, RegressorMixin):
190            self.type_fit = "regression"
191        elif isinstance(self, ClassifierMixin):
192            self.type_fit = "classification"
193        self.subsampler_ = None
194        self.index_col_ = None
195        self.index_row_ = True
196        self.clustering_obj_ = None
197        self.clustering_scaler_ = None
198        self.nn_scaler_ = None
199        self.scaler_ = None
200        self.encoder_ = None
201        self.W_ = None
202        self.X_ = None
203        self.y_ = None
204        self.y_mean_ = None
205        self.beta_ = None
206
207        # activation function -----
208        if sys_platform in ("Linux", "Darwin"):
209            activation_options = {
210                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
211                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
212                "sigmoid": (
213                    ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid
214                ),
215                "prelu": partial(ac.prelu, a=a),
216                "elu": (
217                    partial(ac.elu, a=a)
218                    if (self.backend == "cpu")
219                    else partial(jnn.elu, a=a)
220                ),
221            }
222        else:  # on Windows currently, no JAX
223            activation_options = {
224                "relu": (
225                    ac.relu if (self.backend == "cpu") else NotImplementedError
226                ),
227                "tanh": (
228                    np.tanh if (self.backend == "cpu") else NotImplementedError
229                ),
230                "sigmoid": (
231                    ac.sigmoid
232                    if (self.backend == "cpu")
233                    else NotImplementedError
234                ),
235                "prelu": partial(ac.prelu, a=a),
236                "elu": (
237                    partial(ac.elu, a=a)
238                    if (self.backend == "cpu")
239                    else NotImplementedError
240                ),
241            }
242        self.activation_func = activation_options[activation_name]
243
244    # "preprocessing" methods to be inherited -----
245
246    def encode_clusters(self, X=None, predict=False, scaler=None, **kwargs):  #
247        """Create new covariates with kmeans or GMM clustering
248
249        Parameters:
250
251            X: {array-like}, shape = [n_samples, n_features]
252                Training vectors, where n_samples is the number
253                of samples and n_features is the number of features.
254
255            predict: boolean
256                is False on training set and True on test set
257
258            scaler: {object} of class StandardScaler, MinMaxScaler, RobustScaler or MaxAbsScaler
259                if scaler has already been fitted on training data (online training), it can be passed here
260
261            **kwargs:
262                additional parameters to be passed to the
263                clustering method
264
265        Returns:
266
267            Clusters' matrix, one-hot encoded: {array-like}
268
269        """
270
271        np.random.seed(self.seed)
272
273        if X is None:
274            X = self.X_
275
276        if isinstance(X, pd.DataFrame):
277            X = copy.deepcopy(X.values.astype(float))
278
279        if len(X.shape) == 1:
280            X = X.reshape(1, -1)
281
282        if predict is False:  # encode training set
283
284            # scale input data before clustering
285            self.clustering_scaler_, scaled_X = mo.scale_covariates(
286                X, choice=self.type_scaling[2], scaler=self.clustering_scaler_
287            )
288
289            self.clustering_obj_, X_clustered = mo.cluster_covariates(
290                scaled_X,
291                self.n_clusters,
292                self.seed,
293                type_clust=self.type_clust,
294                **kwargs
295            )
296
297            if self.cluster_encode == True:
298                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
299                    np.float16
300                )
301
302            return X_clustered.astype(np.float16)
303
304        # if predict == True, encode test set
305        X_clustered = self.clustering_obj_.predict(
306            self.clustering_scaler_.transform(X)
307        )
308
309        if self.cluster_encode == True:
310            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
311                np.float16
312            )
313
314        return X_clustered.astype(np.float16)
315
316    def create_layer(self, scaled_X, W=None):
317        """Create hidden layer.
318
319        Parameters:
320
321            scaled_X: {array-like}, shape = [n_samples, n_features]
322                Training vectors, where n_samples is the number
323                of samples and n_features is the number of features
324
325            W: {array-like}, shape = [n_features, hidden_features]
326                if provided, constructs the hidden layer with W; otherwise computed internally
327
328        Returns:
329
330            Hidden layer matrix: {array-like}
331
332        """
333
334        n_features = scaled_X.shape[1]
335
336        # hash_sim = {
337        #         "sobol": generate_sobol,
338        #         "hammersley": generate_hammersley,
339        #         "uniform": generate_uniform,
340        #         "halton": generate_halton
341        #     }
342
343        if self.bias is False:  # no bias term in the hidden layer
344            if W is None:
345                if self.nodes_sim == "sobol":
346                    self.W_ = generate_sobol(
347                        n_dims=n_features,
348                        n_points=self.n_hidden_features,
349                        seed=self.seed,
350                    )
351                elif self.nodes_sim == "hammersley":
352                    self.W_ = generate_hammersley(
353                        n_dims=n_features,
354                        n_points=self.n_hidden_features,
355                        seed=self.seed,
356                    )
357                elif self.nodes_sim == "uniform":
358                    self.W_ = generate_uniform(
359                        n_dims=n_features,
360                        n_points=self.n_hidden_features,
361                        seed=self.seed,
362                    )
363                else:
364                    self.W_ = generate_halton(
365                        n_dims=n_features,
366                        n_points=self.n_hidden_features,
367                        seed=self.seed,
368                    )
369
370                # self.W_ = hash_sim[self.nodes_sim](
371                #             n_dims=n_features,
372                #             n_points=self.n_hidden_features,
373                #             seed=self.seed,
374                #         )
375
376                assert (
377                    scaled_X.shape[1] == self.W_.shape[0]
378                ), "check dimensions of covariates X and matrix W"
379
380                return mo.dropout(
381                    x=self.activation_func(
382                        mo.safe_sparse_dot(
383                            a=scaled_X, b=self.W_, backend=self.backend
384                        )
385                    ),
386                    drop_prob=self.dropout,
387                    seed=self.seed,
388                )
389
390            # W is not none
391            assert (
392                scaled_X.shape[1] == W.shape[0]
393            ), "check dimensions of covariates X and matrix W"
394
395            # self.W_ = W
396            return mo.dropout(
397                x=self.activation_func(
398                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
399                ),
400                drop_prob=self.dropout,
401                seed=self.seed,
402            )
403
404        # with bias term in the hidden layer
405        if W is None:
406            n_features_1 = n_features + 1
407
408            if self.nodes_sim == "sobol":
409                self.W_ = generate_sobol(
410                    n_dims=n_features_1,
411                    n_points=self.n_hidden_features,
412                    seed=self.seed,
413                )
414            elif self.nodes_sim == "hammersley":
415                self.W_ = generate_hammersley(
416                    n_dims=n_features_1,
417                    n_points=self.n_hidden_features,
418                    seed=self.seed,
419                )
420            elif self.nodes_sim == "uniform":
421                self.W_ = generate_uniform(
422                    n_dims=n_features_1,
423                    n_points=self.n_hidden_features,
424                    seed=self.seed,
425                )
426            else:
427                self.W_ = generate_halton(
428                    n_dims=n_features_1,
429                    n_points=self.n_hidden_features,
430                    seed=self.seed,
431                )
432
433            # self.W_ = hash_sim[self.nodes_sim](
434            #         n_dims=n_features_1,
435            #         n_points=self.n_hidden_features,
436            #         seed=self.seed,
437            #     )
438
439            return mo.dropout(
440                x=self.activation_func(
441                    mo.safe_sparse_dot(
442                        a=mo.cbind(
443                            np.ones(scaled_X.shape[0]),
444                            scaled_X,
445                            backend=self.backend,
446                        ),
447                        b=self.W_,
448                        backend=self.backend,
449                    )
450                ),
451                drop_prob=self.dropout,
452                seed=self.seed,
453            )
454
455        # W is not None
456        # self.W_ = W
457        return mo.dropout(
458            x=self.activation_func(
459                mo.safe_sparse_dot(
460                    a=mo.cbind(
461                        np.ones(scaled_X.shape[0]),
462                        scaled_X,
463                        backend=self.backend,
464                    ),
465                    b=W,
466                    backend=self.backend,
467                )
468            ),
469            drop_prob=self.dropout,
470            seed=self.seed,
471        )
472
473    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
474        """Create new hidden features for training set, with hidden layer, center the response.
475
476        Parameters:
477
478            y: array-like, shape = [n_samples]
479                Target values
480
481            X: {array-like}, shape = [n_samples, n_features]
482                Training vectors, where n_samples is the number
483                of samples and n_features is the number of features
484
485            W: {array-like}, shape = [n_features, hidden_features]
486                if provided, constructs the hidden layer via W
487
488        Returns:
489
490            (centered response, direct link + hidden layer matrix): {tuple}
491
492        """
493
494        # either X and y are stored or not
495        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
496        if self.n_hidden_features > 0:  # has a hidden layer
497            assert (
498                len(self.type_scaling) >= 2
499            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
500
501        if X is None:
502
503            if self.col_sample == 1:
504                input_X = self.X_
505            else:
506                n_features = self.X_.shape[1]
507                new_n_features = int(np.ceil(n_features * self.col_sample))
508                assert (
509                    new_n_features >= 1
510                ), "check class attribute 'col_sample' and the number of covariates provided for X"
511                np.random.seed(self.seed)
512                index_col = np.random.choice(
513                    range(n_features), size=new_n_features, replace=False
514                )
515                self.index_col_ = index_col
516                input_X = self.X_[:, self.index_col_]
517
518        else:  # X is not None # keep X vs self.X_
519
520            if isinstance(X, pd.DataFrame):
521                X = copy.deepcopy(X.values.astype(float))
522
523            if self.col_sample == 1:
524                input_X = X
525            else:
526                n_features = X.shape[1]
527                new_n_features = int(np.ceil(n_features * self.col_sample))
528                assert (
529                    new_n_features >= 1
530                ), "check class attribute 'col_sample' and the number of covariates provided for X"
531                np.random.seed(self.seed)
532                index_col = np.random.choice(
533                    range(n_features), size=new_n_features, replace=False
534                )
535                self.index_col_ = index_col
536                input_X = X[:, self.index_col_]
537
538        if self.n_clusters <= 0:
539            # data without any clustering: self.n_clusters is None -----
540
541            if self.n_hidden_features > 0:  # with hidden layer
542
543                self.nn_scaler_, scaled_X = mo.scale_covariates(
544                    input_X, choice=self.type_scaling[1], scaler=self.nn_scaler_
545                )
546                Phi_X = (
547                    self.create_layer(scaled_X)
548                    if W is None
549                    else self.create_layer(scaled_X, W=W)
550                )
551                Z = (
552                    mo.cbind(input_X, Phi_X, backend=self.backend)
553                    if self.direct_link is True
554                    else Phi_X
555                )
556                self.scaler_, scaled_Z = mo.scale_covariates(
557                    Z, choice=self.type_scaling[0], scaler=self.scaler_
558                )
559            else:  # no hidden layer
560                Z = input_X
561                self.scaler_, scaled_Z = mo.scale_covariates(
562                    Z, choice=self.type_scaling[0], scaler=self.scaler_
563                )
564
565        else:
566
567            # data with clustering: self.n_clusters is not None ----- # keep
568
569            augmented_X = mo.cbind(
570                input_X,
571                self.encode_clusters(input_X, **kwargs),
572                backend=self.backend,
573            )
574
575            if self.n_hidden_features > 0:  # with hidden layer
576
577                self.nn_scaler_, scaled_X = mo.scale_covariates(
578                    augmented_X,
579                    choice=self.type_scaling[1],
580                    scaler=self.nn_scaler_,
581                )
582                Phi_X = (
583                    self.create_layer(scaled_X)
584                    if W is None
585                    else self.create_layer(scaled_X, W=W)
586                )
587                Z = (
588                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
589                    if self.direct_link is True
590                    else Phi_X
591                )
592                self.scaler_, scaled_Z = mo.scale_covariates(
593                    Z, choice=self.type_scaling[0], scaler=self.scaler_
594                )
595            else:  # no hidden layer
596                Z = augmented_X
597                self.scaler_, scaled_Z = mo.scale_covariates(
598                    Z, choice=self.type_scaling[0], scaler=self.scaler_
599                )
600
601        # Returning model inputs -----
602        if mx.is_factor(y) is False:  # regression
603            # center y
604            if y is None:
605                self.y_mean_, centered_y = mo.center_response(self.y_)
606            else:
607                self.y_mean_, centered_y = mo.center_response(y)
608
609            # y is subsampled
610            if self.row_sample < 1:
611                n, p = Z.shape
612
613                self.subsampler_ = (
614                    SubSampler(
615                        y=self.y_, row_sample=self.row_sample, seed=self.seed
616                    )
617                    if y is None
618                    else SubSampler(
619                        y=y, row_sample=self.row_sample, seed=self.seed
620                    )
621                )
622
623                self.index_row_ = self.subsampler_.subsample()
624
625                n_row_sample = len(self.index_row_)
626                # regression
627                return (
628                    centered_y[self.index_row_].reshape(n_row_sample),
629                    self.scaler_.transform(
630                        Z[self.index_row_, :].reshape(n_row_sample, p)
631                    ),
632                )
633            # y is not subsampled
634            # regression
635            return (centered_y, self.scaler_.transform(Z))
636
637        # classification
638        # y is subsampled
639        if self.row_sample < 1:
640            n, p = Z.shape
641
642            self.subsampler_ = (
643                SubSampler(
644                    y=self.y_, row_sample=self.row_sample, seed=self.seed
645                )
646                if y is None
647                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
648            )
649
650            self.index_row_ = self.subsampler_.subsample()
651
652            n_row_sample = len(self.index_row_)
653            # classification
654            return (
655                y[self.index_row_].reshape(n_row_sample),
656                self.scaler_.transform(
657                    Z[self.index_row_, :].reshape(n_row_sample, p)
658                ),
659            )
660        # y is not subsampled
661        # classification
662        return (y, self.scaler_.transform(Z))
663
664    def cook_test_set(self, X, **kwargs):
665        """Transform data from test set, with hidden layer.
666
667        Parameters:
668
669            X: {array-like}, shape = [n_samples, n_features]
670                Training vectors, where n_samples is the number
671                of samples and n_features is the number of features
672
673            **kwargs: additional parameters to be passed to self.encode_cluster
674
675        Returns:
676
677            Transformed test set : {array-like}
678        """
679
680        if isinstance(X, pd.DataFrame):
681            X = copy.deepcopy(X.values.astype(float))
682
683        if len(X.shape) == 1:
684            X = X.reshape(1, -1)
685
686        if (
687            self.n_clusters == 0
688        ):  # data without clustering: self.n_clusters is None -----
689            if self.n_hidden_features > 0:
690                # if hidden layer
691                scaled_X = (
692                    self.nn_scaler_.transform(X)
693                    if (self.col_sample == 1)
694                    else self.nn_scaler_.transform(X[:, self.index_col_])
695                )
696                Phi_X = self.create_layer(scaled_X, self.W_)
697                if self.direct_link == True:
698                    return self.scaler_.transform(
699                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
700                    )
701                # when self.direct_link == False
702                return self.scaler_.transform(Phi_X)
703            # if no hidden layer # self.n_hidden_features == 0
704            return self.scaler_.transform(X)
705
706        # data with clustering: self.n_clusters > 0 -----
707        if self.col_sample == 1:
708            predicted_clusters = self.encode_clusters(
709                X=X, predict=True, **kwargs
710            )
711            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
712        else:
713            predicted_clusters = self.encode_clusters(
714                X=X[:, self.index_col_], predict=True, **kwargs
715            )
716            augmented_X = mo.cbind(
717                X[:, self.index_col_], predicted_clusters, backend=self.backend
718            )
719
720        if self.n_hidden_features > 0:  # if hidden layer
721            scaled_X = self.nn_scaler_.transform(augmented_X)
722            Phi_X = self.create_layer(scaled_X, self.W_)
723            if self.direct_link == True:
724                return self.scaler_.transform(
725                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
726                )
727            return self.scaler_.transform(Phi_X)
728
729        # if no hidden layer
730        return self.scaler_.transform(augmented_X)
731
732    def cross_val_score(
733        self,
734        X,
735        y,
736        cv=5,
737        scoring="accuracy",
738        random_state=42,
739        n_jobs=-1,
740        epsilon=0.5,
741        penalized=True,
742        objective="abs",
743        **kwargs
744    ):
745        """
746        Penalized Cross-validation score for a model.
747
748        Parameters:
749
750            X: {array-like}, shape = [n_samples, n_features]
751                Training vectors, where n_samples is the number
752                of samples and n_features is the number of features
753
754            y: array-like, shape = [n_samples]
755                Target values
756
757            X_test: {array-like}, shape = [n_samples, n_features]
758                Test vectors, where n_samples is the number
759                of samples and n_features is the number of features
760
761            y_test: array-like, shape = [n_samples]
762                Target values
763
764            cv: int
765                Number of folds
766
767            scoring: str
768                Scoring metric
769
770            random_state: int
771                Random state
772
773            n_jobs: int
774                Number of jobs to run in parallel
775
776            epsilon: float
777                Penalty parameter
778
779            penalized: bool
780                Whether to obtain penalized cross-validation score or not
781
782            objective: str
783                'abs': Minimize the absolute difference between cross-validation score and validation score
784                'relative': Minimize the relative difference between cross-validation score and validation score
785        Returns:
786
787            A namedtuple with the following fields:
788                - cv_score: float
789                    cross-validation score
790                - val_score: float
791                    validation score
792                - penalized_score: float
793                    penalized cross-validation score: cv_score / val_score + epsilon*(1/val_score + 1/cv_score)
794                    If higher scoring metric is better, minimize the function result.
795                    If lower scoring metric is better, maximize the function result.
796        """
797        if scoring == "accuracy":
798            scoring_func = accuracy_score
799        elif scoring == "balanced_accuracy":
800            scoring_func = balanced_accuracy_score
801        elif scoring == "f1":
802            scoring_func = f1_score
803        elif scoring == "roc_auc":
804            scoring_func = roc_auc_score
805        elif scoring == "r2":
806            scoring_func = r2_score
807        elif scoring == "mse":
808            scoring_func = mean_squared_error
809        elif scoring == "mae":
810            scoring_func = mean_absolute_error
811        elif scoring == "mape":
812            scoring_func = mean_absolute_percentage_error
813        elif scoring == "rmse":
814
815            def scoring_func(y_true, y_pred):
816                return np.sqrt(mean_squared_error(y_true, y_pred))
817
818        X_train, X_val, y_train, y_val = train_test_split(
819            X, y, test_size=0.2, random_state=random_state
820        )
821
822        res = cross_val_score(
823            self, X_train, y_train, cv=cv, scoring=scoring, n_jobs=n_jobs
824        )  # cross-validation error
825
826        if penalized == False:
827            return res
828
829        DescribeResult = namedtuple(
830            "DescribeResult", ["cv_score", "val_score", "penalized_score"]
831        )
832
833        numerator = res.mean()
834
835        # Evaluate on the (cv+1)-th fold
836        preds_val = self.fit(X_train, y_train).predict(X_val)
837        try:
838            denominator = scoring(y_val, preds_val)  # validation error
839        except Exception as e:
840            denominator = scoring_func(y_val, preds_val)
841
842        # if higher is better
843        if objective == "abs":
844            penalized_score = np.abs(numerator - denominator) + epsilon * (
845                1 / denominator + 1 / numerator
846            )
847        elif objective == "relative":
848            ratio = numerator / denominator
849            penalized_score = np.abs(ratio - 1) + epsilon * (
850                1 / denominator + 1 / numerator
851            )
852
853        return DescribeResult(
854            cv_score=numerator,
855            val_score=denominator,
856            penalized_score=penalized_score,
857        )