Skip to content

Commit

Permalink
Updating search space (#156)
Browse files Browse the repository at this point in the history 
* Updating search space

* fix typo

* Bug fix

* Fixing buggy implementation of predict when using gpu
  • Loading branch information
@ArlindKadra
ArlindKadra authored Apr 6, 2021
1 parent f4c1fe2 commit 40b49fe
Show file tree
Hide file tree
Showing 11 changed files with 38 additions and 24 deletions.
2 changes: 1 addition & 1 deletion autoPyTorch/datasets/resampling_strategy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def holdout_validation(val_share: float, indices: np.ndarray, **kwargs: Any) ->

def stratified_holdout_validation(val_share: float, indices: np.ndarray, **kwargs: Any) \
-> Tuple[np.ndarray, np.ndarray]:
train, val = train_test_split(indices, test_size=val_share, shuffle=False, stratify=kwargs["stratify"])
train, val = train_test_split(indices, test_size=val_share, shuffle=True, stratify=kwargs["stratify"])
return train, val


Expand Down
5 changes: 3 additions & 2 deletions autoPyTorch/pipeline/components/setup/network/base_network.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ def predict(self, loader: torch.utils.data.DataLoader) -> torch.Tensor:
return Y_snapshot_preds_tensor.mean(dim=0).cpu().numpy()

def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader) -> torch.Tensor:
network.to(self.device)
network.float()
network.eval()
# Batch prediction
Expand All @@ -133,10 +134,10 @@ def _predict(self, network: torch.nn.Module, loader: torch.utils.data.DataLoader
for i, (X_batch, Y_batch) in enumerate(loader):
# Predict on batch
X_batch = X_batch.float().to(self.device)
Y_batch_pred = network(X_batch).detach().cpu()
Y_batch_pred = network(X_batch)
if self.final_activation is not None:
Y_batch_pred = self.final_activation(Y_batch_pred)
Y_batch_preds.append(Y_batch_pred)
Y_batch_preds.append(Y_batch_pred.detach().cpu())

return torch.cat(Y_batch_preds, 0)

Expand Down
5 changes: 3 additions & 2 deletions autoPyTorch/pipeline/components/setup/network_backbone/MLPBackbone.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
list(_activations.keys())[0]),
use_dropout: Tuple[Tuple, bool] = ((True, False), False),
num_units: Tuple[Tuple, int] = ((10, 1024), 200),
num_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True),
dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5)
) -> ConfigurationSpace:

Expand Down Expand Up @@ -102,7 +102,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
n_units_hp = UniformIntegerHyperparameter("num_units_%d" % i,
lower=num_units[0][0],
upper=num_units[0][1],
default_value=num_units[1])
default_value=num_units[1],
log=num_units[2],)
cs.add_hyperparameter(n_units_hp)

if i > min_mlp_layers:
Expand Down
5 changes: 3 additions & 2 deletions autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
use_batch_norm: Tuple[Tuple, bool] = ((True, False), True),
use_dropout: Tuple[Tuple, bool] = ((True, False), False),
use_skip_connection: Tuple[Tuple, bool] = ((True, False), True),
num_units: Tuple[Tuple, int] = ((10, 1024), 200),
num_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True),
activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
list(_activations.keys())[0]),
blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2),
Expand Down Expand Up @@ -162,7 +162,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
"num_units_%d" % i,
lower=num_units[0][0],
upper=num_units[0][1],
default_value=num_units[1]
default_value=num_units[1],
log=num_units[2],
)
blocks_per_group = UniformIntegerHyperparameter(
"blocks_per_group_%d" % i,
Expand Down
9 changes: 6 additions & 3 deletions autoPyTorch/pipeline/components/setup/network_backbone/ShapedMLPBackbone.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
num_groups: Tuple[Tuple, int] = ((1, 15), 5),
max_dropout: Tuple[Tuple, float] = ((0, 1), 0.5),
use_dropout: Tuple[Tuple, bool] = ((True, False), False),
max_units: Tuple[Tuple, int] = ((10, 1024), 200),
output_dim: Tuple[Tuple, int] = ((10, 1024), 200),
max_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True),
output_dim: Tuple[Tuple, int, bool] = ((10, 1024), 200, True),
mlp_shape: Tuple[Tuple, str] = (('funnel', 'long_funnel',
'diamond', 'hexagon',
'brick', 'triangle', 'stairs'), 'funnel'),
Expand All @@ -104,18 +104,21 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
default_value=activation[1]
)
(min_num_units, max_num_units), default_units = max_units[:2]
log_nr_max_units = max_units[2]
max_units = UniformIntegerHyperparameter(
"max_units",
lower=min_num_units,
upper=max_num_units,
default_value=default_units,
log=log_nr_max_units,
)

output_dim = UniformIntegerHyperparameter(
"output_dim",
lower=output_dim[0][0],
upper=output_dim[0][1],
default_value=output_dim[1]
default_value=output_dim[1],
log=output_dim[2],
)

cs.add_hyperparameters([num_groups, activation, mlp_shape, max_units, output_dim])
Expand Down
11 changes: 7 additions & 4 deletions autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
use_batch_norm: Tuple[Tuple, bool] = ((True, False), True),
use_dropout: Tuple[Tuple, bool] = ((True, False), False),
use_skip_connection: Tuple[Tuple, bool] = ((True, False), True),
max_units: Tuple[Tuple, int] = ((10, 1024), 200),
max_units: Tuple[Tuple, int, bool] = ((10, 1024), 200, True),
blocks_per_group: Tuple[Tuple, int] = ((1, 4), 2),
max_dropout: Tuple[Tuple, float] = ((0, 0.8), 0.5),
multi_branch_choice: Tuple[Tuple, str] = (('shake-drop', 'shake-shake',
Expand All @@ -96,7 +96,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
'brick', 'triangle', 'stairs'), 'funnel'),
activation: Tuple[Tuple, str] = (
tuple(_activations.keys()), list(_activations.keys())[0]),
output_dim: Tuple[Tuple, int] = ((10, 1024), 200),
output_dim: Tuple[Tuple, int, bool] = ((10, 1024), 200, True),
) -> ConfigurationSpace:
cs = ConfigurationSpace()

Expand Down Expand Up @@ -124,11 +124,13 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
default_value=activation[1]
)
(min_num_units, max_num_units), default_units = max_units[:2]
log_num_units = max_units[2]
output_dim = UniformIntegerHyperparameter(
"output_dim",
lower=output_dim[0][0],
upper=output_dim[0][1],
default_value=output_dim[1]
default_value=output_dim[1],
log=output_dim[2],
)

cs.add_hyperparameters([num_groups, blocks_per_group, activation, output_dim])
Expand Down Expand Up @@ -161,7 +163,8 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
"max_units",
lower=min_num_units,
upper=max_num_units,
default_value=default_units
default_value=default_units,
log=log_num_units,
)
cs.add_hyperparameters([max_units])

Expand Down
4 changes: 2 additions & 2 deletions autoPyTorch/pipeline/components/setup/optimizer/AdamOptimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9),
beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9),
use_weight_decay: Tuple[Tuple, bool] = ((True, False), True),
weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0)
weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True)
) -> ConfigurationSpace:

cs = ConfigurationSpace()
Expand All @@ -105,7 +105,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
)

weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1],
default_value=weight_decay[1])
default_value=weight_decay[1], log=weight_decay[2])

cs.add_hyperparameters([lr, beta1, beta2, use_wd, weight_decay])

Expand Down
4 changes: 2 additions & 2 deletions autoPyTorch/pipeline/components/setup/optimizer/AdamWOptimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
beta1: Tuple[Tuple, float] = ((0.85, 0.999), 0.9),
beta2: Tuple[Tuple, float] = ((0.9, 0.9999), 0.9),
use_weight_decay: Tuple[Tuple, bool] = ((True, False), True),
weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0)
weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True)
) -> ConfigurationSpace:

cs = ConfigurationSpace()
Expand All @@ -105,7 +105,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
)

weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1],
default_value=weight_decay[1])
default_value=weight_decay[1], log=weight_decay[2])

cs.add_hyperparameters([lr, beta1, beta2, weight_decay, use_wd])

Expand Down
4 changes: 2 additions & 2 deletions autoPyTorch/pipeline/components/setup/optimizer/RMSpropOptimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True),
alpha: Tuple[Tuple, float] = ((0.1, 0.99), 0.99),
use_weight_decay: Tuple[Tuple, bool] = ((True, False), True),
weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0),
weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True),
momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0),
) -> ConfigurationSpace:

Expand All @@ -105,7 +105,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
)

weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1],
default_value=weight_decay[1])
default_value=weight_decay[1], log=weight_decay[2])

momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1],
default_value=momentum[1])
Expand Down
4 changes: 2 additions & 2 deletions autoPyTorch/pipeline/components/setup/optimizer/SGDOptimizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
lr: Tuple[Tuple, float, bool] = ((1e-5, 1e-1), 1e-2, True),
use_weight_decay: Tuple[Tuple, bool] = ((True, False), True),
weight_decay: Tuple[Tuple, float] = ((0.0, 0.1), 0.0),
weight_decay: Tuple[Tuple, float, bool] = ((1E-7, 0.1), 1E-4, True),
momentum: Tuple[Tuple, float] = ((0.0, 0.99), 0.0),
) -> ConfigurationSpace:

Expand All @@ -94,7 +94,7 @@ def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
)

weight_decay = UniformFloatHyperparameter('weight_decay', lower=weight_decay[0][0], upper=weight_decay[0][1],
default_value=weight_decay[1])
default_value=weight_decay[1], log=weight_decay[2])

momentum = UniformFloatHyperparameter('momentum', lower=momentum[0][0], upper=momentum[0][1],
default_value=momentum[1])
Expand Down
9 changes: 7 additions & 2 deletions autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,10 +249,15 @@ def get_torchvision_datasets(self) -> Dict[str, torchvision.datasets.VisionDatas

@staticmethod
def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
batch_size: Tuple[Tuple, int] = ((32, 320), 64)
batch_size: Tuple[Tuple, int, bool] = ((32, 320), 64, True)
) -> ConfigurationSpace:
batch_size = UniformIntegerHyperparameter(
"batch_size", batch_size[0][0], batch_size[0][1], default_value=batch_size[1])
"batch_size",
batch_size[0][0],
batch_size[0][1],
default_value=batch_size[1],
log=batch_size[2],
)
cs = ConfigurationSpace()
cs.add_hyperparameters([batch_size])
return cs
Expand Down

0 comments on commit 40b49fe

Please sign in to comment.