diff --git a/autoPyTorch/datasets/resampling_strategy.py b/autoPyTorch/datasets/resampling_strategy.py index 1d0bc3077..b853fac0a 100644 --- a/autoPyTorch/datasets/resampling_strategy.py +++ b/autoPyTorch/datasets/resampling_strategy.py @@ -97,7 +97,7 @@ def holdout_validation(val_share: float, indices: np.ndarray, **kwargs: Any) -> def stratified_holdout_validation(val_share: float, indices: np.ndarray, **kwargs: Any) \ -> Tuple[np.ndarray, np.ndarray]: - train, val = train_test_split(indices, test_size=val_share, shuffle=False, stratify=kwargs["stratify"]) + train, val = train_test_split(indices, test_size=val_share, shuffle=True, stratify=kwargs["stratify"]) return train, val diff --git a/autoPyTorch/pipeline/components/setup/network/base_network.py b/autoPyTorch/pipeline/components/setup/network/base_network.py index f38ddea60..49306cb6c 100644 --- a/autoPyTorch/pipeline/components/setup/network/base_network.py +++ b/autoPyTorch/pipeline/components/setup/network/base_network.py @@ -125,6 +125,7 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor: return Y_snapshot_preds_tensor.mean(dim=0).cpu().numpy() def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor: + network.to(self.device) network.float() network.eval() # Batch prediction @@ -133,10 +134,10 @@ def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader for i, (X_batch, Y_batch) in enumerate(loader): # Predict on batch X_batch = X_batch.float().to(self.device) - Y_batch_pred = network(X_batch).detach().cpu() + Y_batch_pred = network(X_batch) if self.final_activation is not None: Y_batch_pred = self.final_activation(Y_batch_pred) - Y_batch_preds.append(Y_batch_pred) + Y_batch_preds.append(Y_batch_pred.detach().cpu()) return torch.cat(Y_batch_preds, 0) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py index 230ddfe96..8213f2e28 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py @@ -73,7 +73,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, activation: Tuple[Tuple, str] = (tuple(_activations.keys()), list(_activations.keys())[0]), use_dropout: Tuple[Tuple, bool] = ((True, False), False), - num_units: Tuple[Tuple, int] = ((10, 1024), 200), + num_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True), dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5) ) -> ConfigurationSpace: @@ -102,7 +102,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, n_units_hp = UniformIntegerHyperparameter("num_units_%d" % i, lower=num_units[0][0], upper=num_units[0][1], - default_value=num_units[1]) + default_value=num_units[1], + log=num_units[2],) cs.add_hyperparameter(n_units_hp) if i > min_mlp_layers: diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py index cf95ec460..a56260719 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py @@ -97,7 +97,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, use_batch_norm: Tuple[Tuple, bool] = ((True, False), True), use_dropout: Tuple[Tuple, bool] = ((True, False), False), use_skip_connection: Tuple[Tuple, bool] = ((True, False), True), - num_units: Tuple[Tuple, int] = ((10, 1024), 200), + num_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True), activation: Tuple[Tuple, str] = (tuple(_activations.keys()), list(_activations.keys())[0]), blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2), @@ -162,7 +162,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, "num_units_%d" % i, lower=num_units[0][0], upper=num_units[0][1], - default_value=num_units[1] + default_value=num_units[1], + log=num_units[2], ) blocks_per_group = UniformIntegerHyperparameter( "blocks_per_group_%d" % i, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py index 3e8be6b70..38b8e7542 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py @@ -79,8 +79,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, num_groups: Tuple[Tuple, int] = ((1, 15), 5), max_dropout: Tuple[Tuple, float] = ((0, 1), 0.5), use_dropout: Tuple[Tuple, bool] = ((True, False), False), - max_units: Tuple[Tuple, int] = ((10, 1024), 200), - output_dim: Tuple[Tuple, int] = ((10, 1024), 200), + max_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True), + output_dim: Tuple[Tuple, int, bool] = ((10, 1024), 200, True), mlp_shape: Tuple[Tuple, str] = (('funnel', 'long_funnel', 'diamond', 'hexagon', 'brick', 'triangle', 'stairs'), 'funnel'), @@ -104,18 +104,21 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, default_value=activation[1] ) (min_num_units, max_num_units), default_units = max_units[:2] + log_nr_max_units = max_units[2] max_units = UniformIntegerHyperparameter( "max_units", lower=min_num_units, upper=max_num_units, default_value=default_units, + log=log_nr_max_units, ) output_dim = UniformIntegerHyperparameter( "output_dim", lower=output_dim[0][0], upper=output_dim[0][1], - default_value=output_dim[1] + default_value=output_dim[1], + log=output_dim[2], ) cs.add_hyperparameters([num_groups, activation, mlp_shape, max_units, output_dim]) diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py index 9e95a21e2..6a9c463a2 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py @@ -85,7 +85,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, use_batch_norm: Tuple[Tuple, bool] = ((True, False), True), use_dropout: Tuple[Tuple, bool] = ((True, False), False), use_skip_connection: Tuple[Tuple, bool] = ((True, False), True), - max_units: Tuple[Tuple, int] = ((10, 1024), 200), + max_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True), blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2), max_dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5), multi_branch_choice: Tuple[Tuple, str] = (('shake-drop', 'shake-shake', @@ -96,7 +96,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, 'brick', 'triangle', 'stairs'), 'funnel'), activation: Tuple[Tuple, str] = ( tuple(_activations.keys()), list(_activations.keys())[0]), - output_dim: Tuple[Tuple, int] = ((10, 1024), 200), + output_dim: Tuple[Tuple, int, bool] = ((10, 1024), 200, True), ) -> ConfigurationSpace: cs = ConfigurationSpace() @@ -124,11 +124,13 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, default_value=activation[1] ) (min_num_units, max_num_units), default_units = max_units[:2] + log_num_units = max_units[2] output_dim = UniformIntegerHyperparameter( "output_dim", lower=output_dim[0][0], upper=output_dim[0][1], - default_value=output_dim[1] + default_value=output_dim[1], + log=output_dim[2], ) cs.add_hyperparameters([num_groups, blocks_per_group, activation, output_dim]) @@ -161,7 +163,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, "max_units", lower=min_num_units, upper=max_num_units, - default_value=default_units + default_value=default_units, + log=log_num_units, ) cs.add_hyperparameters([max_units]) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py index 90be878c0..2318f1422 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py @@ -83,7 +83,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9), beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9), use_weight_decay: Tuple[Tuple, bool] = ((True, False), True), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0) + weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True) ) -> ConfigurationSpace: cs = ConfigurationSpace() @@ -105,7 +105,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, ) weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) + default_value=weight_decay[1], log=weight_decay[2]) cs.add_hyperparameters([lr, beta1, beta2, use_wd, weight_decay]) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py index 7fdc85f7f..89e9e712c 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py @@ -83,7 +83,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9), beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9), use_weight_decay: Tuple[Tuple, bool] = ((True, False), True), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0) + weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True) ) -> ConfigurationSpace: cs = ConfigurationSpace() @@ -105,7 +105,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, ) weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) + default_value=weight_decay[1], log=weight_decay[2]) cs.add_hyperparameters([lr, beta1, beta2, weight_decay, use_wd]) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py index c60283426..5be33eee4 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py @@ -85,7 +85,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), alpha: Tuple[Tuple, float] = ((0.1, 0.99), 0.99), use_weight_decay: Tuple[Tuple, bool] = ((True, False), True), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0), + weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True), momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0), ) -> ConfigurationSpace: @@ -105,7 +105,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, ) weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) + default_value=weight_decay[1], log=weight_decay[2]) momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1], default_value=momentum[1]) diff --git a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py index 98e271612..b39e37163 100644 --- a/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py +++ b/autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py @@ -77,7 +77,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True), use_weight_decay: Tuple[Tuple, bool] = ((True, False), True), - weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0), + weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True), momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0), ) -> ConfigurationSpace: @@ -94,7 +94,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, ) weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1], - default_value=weight_decay[1]) + default_value=weight_decay[1], log=weight_decay[2]) momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1], default_value=momentum[1]) diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 95a4fccb2..d7b047e27 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -249,10 +249,15 @@ def get_torchvision_datasets(self) -> Dict[str, torchvision.datasets.VisionDatas @staticmethod def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None, - batch_size: Tuple[Tuple, int] = ((32, 320), 64) + batch_size: Tuple[Tuple, int, bool] = ((32, 320), 64, True) ) -> ConfigurationSpace: batch_size = UniformIntegerHyperparameter( - "batch_size", batch_size[0][0], batch_size[0][1], default_value=batch_size[1]) + "batch_size", + batch_size[0][0], + batch_size[0][1], + default_value=batch_size[1], + log=batch_size[2], + ) cs = ConfigurationSpace() cs.add_hyperparameters([batch_size]) return cs