diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index a98cfdd1c..d88a2d3dc 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -941,18 +941,15 @@ def run_traditional_ml( learning algorithm runs over the time limit. """ assert self._logger is not None # for mypy compliancy - if STRING_TO_TASK_TYPES[self.task_type] in REGRESSION_TASKS: - self._logger.warning("Traditional Pipeline is not enabled for regression. Skipping...") - else: - traditional_task_name = 'runTraditional' - self._stopwatch.start_task(traditional_task_name) - elapsed_time = self._stopwatch.wall_elapsed(current_task_name) - time_for_traditional = int(runtime_limit - elapsed_time) - self._do_traditional_prediction( - func_eval_time_limit_secs=func_eval_time_limit_secs, - time_left=time_for_traditional, - ) - self._stopwatch.stop_task(traditional_task_name) + traditional_task_name = 'runTraditional' + self._stopwatch.start_task(traditional_task_name) + elapsed_time = self._stopwatch.wall_elapsed(current_task_name) + time_for_traditional = int(runtime_limit - elapsed_time) + self._do_traditional_prediction( + func_eval_time_limit_secs=func_eval_time_limit_secs, + time_left=time_for_traditional, + ) + self._stopwatch.stop_task(traditional_task_name) def _search( self, @@ -1322,22 +1319,7 @@ def _search( self._logger.info("Starting Shutdown") if proc_ensemble is not None: - self._results_manager.ensemble_performance_history = list(proc_ensemble.history) - - if len(proc_ensemble.futures) > 0: - # Also add ensemble runs that did not finish within smac time - # and add them into the ensemble history - self._logger.info("Ensemble script still running, waiting for it to finish.") - result = proc_ensemble.futures.pop().result() - if result: - ensemble_history, _, _, _ = result - self._results_manager.ensemble_performance_history.extend(ensemble_history) - self._logger.info("Ensemble script finished, continue shutdown.") - - # save the ensemble performance history file - if len(self.ensemble_performance_history) > 0: - pd.DataFrame(self.ensemble_performance_history).to_json( - os.path.join(self._backend.internals_directory, 'ensemble_history.json')) + self._collect_results_ensemble(proc_ensemble) if load_models: self._logger.info("Loading models...") @@ -1605,7 +1587,7 @@ def fit_pipeline( exclude=self.exclude_components, search_space_updates=self.search_space_updates) dataset_properties = dataset.get_dataset_properties(dataset_requirements) - self._backend.replace_datamanager(dataset) + self._backend.save_datamanager(dataset) if self._logger is None: self._logger = self._get_logger(dataset.dataset_name) @@ -1796,7 +1778,7 @@ def fit_ensemble( ensemble_fit_task_name = 'EnsembleFit' self._stopwatch.start_task(ensemble_fit_task_name) if enable_traditional_pipeline: - if func_eval_time_limit_secs is None or func_eval_time_limit_secs > time_for_task: + if func_eval_time_limit_secs > time_for_task: self._logger.warning( 'Time limit for a single run is higher than total time ' 'limit. Capping the limit for a single run to the total ' @@ -1837,12 +1819,8 @@ def fit_ensemble( ) manager.build_ensemble(self._dask_client) - future = manager.futures.pop() - result = future.result() - if result is None: - raise ValueError("Errors occurred while building the ensemble - please" - " check the log file and command line output for error messages.") - self.ensemble_performance_history, _, _, _ = result + if manager is not None: + self._collect_results_ensemble(manager) if load_models: self._load_models() @@ -1920,6 +1898,31 @@ def _init_ensemble_builder( return proc_ensemble + def _collect_results_ensemble( + self, + manager: EnsembleBuilderManager + ) -> None: + + if self._logger is None: + raise ValueError("logger should be initialized to fit ensemble") + + self._results_manager.ensemble_performance_history = list(manager.history) + + if len(manager.futures) > 0: + # Also add ensemble runs that did not finish within smac time + # and add them into the ensemble history + self._logger.info("Ensemble script still running, waiting for it to finish.") + result = manager.futures.pop().result() + if result: + ensemble_history, _, _, _ = result + self._results_manager.ensemble_performance_history.extend(ensemble_history) + self._logger.info("Ensemble script finished, continue shutdown.") + + # save the ensemble performance history file + if len(self.ensemble_performance_history) > 0: + pd.DataFrame(self.ensemble_performance_history).to_json( + os.path.join(self._backend.internals_directory, 'ensemble_history.json')) + def predict( self, X_test: np.ndarray, diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index dc411ea93..a3b710125 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -18,6 +18,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.datasets.resampling_strategy import ( HoldoutValTypes, + CrossValTypes, ResamplingStrategies, ) from autoPyTorch.datasets.tabular_dataset import TabularDataset @@ -437,6 +438,7 @@ def search( if self.dataset is None: raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) + return self._search( dataset=self.dataset, optimize_metric=optimize_metric, @@ -476,23 +478,23 @@ def predict( raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") - X_test = self.input_validator.feature_validator.transform(X_test) + X_test = self.InputValidator.feature_validator.transform(X_test) predicted_probabilities = super().predict(X_test, batch_size=batch_size, n_jobs=n_jobs) - if self.input_validator.target_validator.is_single_column_target(): + if self.InputValidator.target_validator.is_single_column_target(): predicted_indexes = np.argmax(predicted_probabilities, axis=1) else: predicted_indexes = (predicted_probabilities > 0.5).astype(int) # Allow to predict in the original domain -- that is, the user is not interested # in our encoded values - return self.input_validator.target_validator.inverse_transform(predicted_indexes) + return self.InputValidator.target_validator.inverse_transform(predicted_indexes) def predict_proba(self, X_test: Union[np.ndarray, pd.DataFrame, List], batch_size: Optional[int] = None, n_jobs: int = 1) -> np.ndarray: - if self.input_validator is None or not self.input_validator._is_fitted: + if self.InputValidator is None or not self.InputValidator._is_fitted: raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") X_test = self.input_validator.feature_validator.transform(X_test) diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index c616dedb7..ffffbf817 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -18,6 +18,7 @@ from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.datasets.resampling_strategy import ( HoldoutValTypes, + CrossValTypes, ResamplingStrategies, ) from autoPyTorch.datasets.tabular_dataset import TabularDataset @@ -437,6 +438,7 @@ def search( if self.dataset is None: raise ValueError("`dataset` in {} must be initialized, but got None".format(self.__class__.__name__)) + return self._search( dataset=self.dataset, optimize_metric=optimize_metric, @@ -462,14 +464,14 @@ def predict( batch_size: Optional[int] = None, n_jobs: int = 1 ) -> np.ndarray: - if self.input_validator is None or not self.input_validator._is_fitted: + if self.InputValidator is None or not self.InputValidator._is_fitted: raise ValueError("predict() is only supported after calling search. Kindly call first " "the estimator search() method.") - X_test = self.input_validator.feature_validator.transform(X_test) + X_test = self.InputValidator.feature_validator.transform(X_test) predicted_values = super().predict(X_test, batch_size=batch_size, n_jobs=n_jobs) # Allow to predict in the original domain -- that is, the user is not interested # in our encoded values - return self.input_validator.target_validator.inverse_transform(predicted_values) + return self.InputValidator.target_validator.inverse_transform(predicted_values) diff --git a/autoPyTorch/data/base_target_validator.py b/autoPyTorch/data/base_target_validator.py index cd1f5cd54..ae9e940c5 100644 --- a/autoPyTorch/data/base_target_validator.py +++ b/autoPyTorch/data/base_target_validator.py @@ -86,7 +86,6 @@ def fit( np.shape(y_test) )) if isinstance(y_train, pd.DataFrame): - y_train = cast(pd.DataFrame, y_train) y_test = cast(pd.DataFrame, y_test) if y_train.columns.tolist() != y_test.columns.tolist(): raise ValueError( diff --git a/autoPyTorch/data/tabular_feature_validator.py b/autoPyTorch/data/tabular_feature_validator.py index ffb1128f4..ef936d6f1 100644 --- a/autoPyTorch/data/tabular_feature_validator.py +++ b/autoPyTorch/data/tabular_feature_validator.py @@ -2,6 +2,7 @@ from logging import Logger from typing import Dict, List, Optional, Tuple, Union, cast + import numpy as np import pandas as pd @@ -270,7 +271,7 @@ def transform( if isinstance(X, np.ndarray): X = self.numpy_to_pandas(X) - if hasattr(X, "iloc") and not issparse(X): + if ispandas(X) and not issparse(X): X = cast(pd.DataFrame, X) # Check the data here so we catch problems on new test data @@ -400,9 +401,6 @@ def _get_columns_info( Type of each column numerical/categorical """ - if len(self.transformed_columns) > 0 and self.feat_type is not None: - return self.transformed_columns, self.feat_type - # Register if a column needs encoding numerical_columns = [] categorical_columns = [] diff --git a/autoPyTorch/data/tabular_target_validator.py b/autoPyTorch/data/tabular_target_validator.py index 22cabb999..a419034af 100644 --- a/autoPyTorch/data/tabular_target_validator.py +++ b/autoPyTorch/data/tabular_target_validator.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union, cast +from typing import List, Optional, cast import numpy as np diff --git a/autoPyTorch/evaluation/fit_evaluator.py b/autoPyTorch/evaluation/fit_evaluator.py index 281913003..f171cc18c 100644 --- a/autoPyTorch/evaluation/fit_evaluator.py +++ b/autoPyTorch/evaluation/fit_evaluator.py @@ -10,13 +10,13 @@ from smac.tae import StatusType +from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes from autoPyTorch.evaluation.abstract_evaluator import ( AbstractEvaluator, fit_and_suppress_warnings ) from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric -from autoPyTorch.utils.backend import Backend from autoPyTorch.utils.common import subsampler from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates diff --git a/autoPyTorch/optimizer/smbo.py b/autoPyTorch/optimizer/smbo.py index 898afd7f5..d790237b7 100644 --- a/autoPyTorch/optimizer/smbo.py +++ b/autoPyTorch/optimizer/smbo.py @@ -104,7 +104,7 @@ def __init__(self, resampling_strategy_args: Optional[Dict[str, Any]] = None, include: Optional[Dict[str, Any]] = None, exclude: Optional[Dict[str, Any]] = None, - disable_file_output: List = [], + disable_file_output: Union[bool, List[str]] = False, smac_scenario_args: Optional[Dict[str, Any]] = None, get_smac_object_callback: Optional[Callable] = None, all_supported_metrics: bool = True, diff --git a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py index 57304a6e4..1af7ad7af 100644 --- a/autoPyTorch/pipeline/components/setup/network_backbone/utils.py +++ b/autoPyTorch/pipeline/components/setup/network_backbone/utils.py @@ -83,11 +83,7 @@ class ShakeDropFunction(Function): Github URL: https://github.com/owruby/shake-drop_pytorch/blob/master/models/shakedrop.py """ @staticmethod -<<<<<<< HEAD def forward(ctx: Any, -======= - def forward(ctx: typing.Any, ->>>>>>> Bug fixes (#249) x: torch.Tensor, alpha: torch.Tensor, beta: torch.Tensor, @@ -114,31 +110,20 @@ def backward(ctx: Any, shake_drop = ShakeDropFunction.apply -<<<<<<< HEAD -def shake_get_alpha_beta(is_training: bool, is_cuda: bool - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - The methods used in this function have been introduced in 'ShakeShake Regularisation' - Currently, this function supports `shake-shake`. -======= def shake_get_alpha_beta( is_training: bool, is_cuda: bool, method: str -) -> typing.Tuple[torch.Tensor, torch.Tensor]: +) -> Tuple[torch.Tensor, torch.Tensor]: """ The methods used in this function have been introduced in 'ShakeShake Regularisation' Each method name is available in the referred paper. Currently, this function supports `even-even`, `shake-even`, `shake-shake` and `M3`. ->>>>>>> Bug fixes (#249) Args: is_training (bool): Whether the computation for the training is_cuda (bool): Whether the tensor is on CUDA -<<<<<<< HEAD -======= method (str): The shake method either `even-even`, `shake-even`, `shake-shake` or `M3` ->>>>>>> Bug fixes (#249) Returns: alpha, beta (Tuple[float, float]): @@ -150,14 +135,8 @@ def shake_get_alpha_beta( Author: Xavier Gastaldi URL: https://arxiv.org/abs/1705.07485 -<<<<<<< HEAD - Note: - The names have been taken from the paper as well. - Currently, this function supports `shake-shake`. -======= The names have been taken from the paper as well. Currently, this function supports `even-even`, `shake-even`, `shake-shake` and `M3`. ->>>>>>> Bug fixes (#249) """ if not is_training: result = (torch.FloatTensor([0.5]), torch.FloatTensor([0.5])) @@ -187,27 +166,15 @@ def shake_get_alpha_beta( def shake_drop_get_bl( -<<<<<<< HEAD - block_index: int, - min_prob_no_shake: float, - num_blocks: int, - is_training: bool, - is_cuda: bool -======= block_index: int, min_prob_no_shake: float, num_blocks: int, is_training: bool, is_cuda: bool ->>>>>>> Bug fixes (#249) ) -> torch.Tensor: """ The sampling of Bernoulli random variable based on Eq. (4) in the paper -<<<<<<< HEAD - -======= ->>>>>>> Bug fixes (#249) Args: block_index (int): The index of the block from the input layer min_prob_no_shake (float): The initial shake probability @@ -217,28 +184,16 @@ def shake_drop_get_bl( Returns: bl (torch.Tensor): a Bernoulli random variable in {0, 1} -<<<<<<< HEAD - -======= ->>>>>>> Bug fixes (#249) Reference: ShakeDrop Regularization for Deep Residual Learning Yoshihiro Yamada et. al. (2020) paper: https://arxiv.org/pdf/1802.02375.pdf implementation: https://github.com/imenurok/ShakeDrop """ -<<<<<<< HEAD - - pl = 1 - ((block_index + 1) / num_blocks) * (1 - min_prob_no_shake) - - if is_training: - # Move to torch.rand(1) for reproducibility -======= pl = 1 - ((block_index + 1) / num_blocks) * (1 - min_prob_no_shake) if is_training: # Move to torch.randn(1) for reproducibility ->>>>>>> Bug fixes (#249) bl = torch.as_tensor(1.0) if torch.rand(1) <= pl else torch.as_tensor(0.0) else: bl = torch.as_tensor(pl) diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py index 381e0735d..d59597040 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py @@ -148,62 +148,71 @@ def get_hyperparameter_search_space( if default is None: defaults = [ 'NoEmbedding', - 'LearnedEntityEmbedding', + # 'LearnedEntityEmbedding', ] for default_ in defaults: if default_ in available_embedding: default = default_ break - categorical_columns = dataset_properties['categorical_columns'] \ - if isinstance(dataset_properties['categorical_columns'], List) else [] - - updates = self._get_search_space_updates() - if '__choice__' in updates.keys(): - choice_hyperparameter = updates['__choice__'] - if not set(choice_hyperparameter.value_range).issubset(available_embedding): - raise ValueError("Expected given update for {} to have " - "choices in {} got {}".format(self.__class__.__name__, - available_embedding, - choice_hyperparameter.value_range)) - if len(categorical_columns) == 0: - assert len(choice_hyperparameter.value_range) == 1 - if 'NoEmbedding' not in choice_hyperparameter.value_range: - raise ValueError("Provided {} in choices, however, the dataset " - "is incompatible with it".format(choice_hyperparameter.value_range)) - embedding = CSH.CategoricalHyperparameter('__choice__', - choice_hyperparameter.value_range, - default_value=choice_hyperparameter.default_value) - else: - - if len(categorical_columns) == 0: - default = 'NoEmbedding' - if include is not None and default not in include: - raise ValueError("Provided {} in include, however, the dataset " - "is incompatible with it".format(include)) - embedding = CSH.CategoricalHyperparameter('__choice__', - ['NoEmbedding'], - default_value=default) - else: - embedding = CSH.CategoricalHyperparameter('__choice__', - list(available_embedding.keys()), - default_value=default) - + # Restrict embedding to NoEmbedding until preprocessing is fixed + embedding = CSH.CategoricalHyperparameter('__choice__', + ['NoEmbedding'], + default_value=default) cs.add_hyperparameter(embedding) - for name in embedding.choices: - updates = self._get_search_space_updates(prefix=name) - config_space = available_embedding[name].get_hyperparameter_search_space(dataset_properties, # type: ignore - **updates) - parent_hyperparameter = {'parent': embedding, 'value': name} - cs.add_configuration_space( - name, - config_space, - parent_hyperparameter=parent_hyperparameter - ) - self.configuration_space_ = cs self.dataset_properties_ = dataset_properties return cs + # categorical_columns = dataset_properties['categorical_columns'] \ + # if isinstance(dataset_properties['categorical_columns'], List) else [] + + # updates = self._get_search_space_updates() + # if '__choice__' in updates.keys(): + # choice_hyperparameter = updates['__choice__'] + # if not set(choice_hyperparameter.value_range).issubset(available_embedding): + # raise ValueError("Expected given update for {} to have " + # "choices in {} got {}".format(self.__class__.__name__, + # available_embedding, + # choice_hyperparameter.value_range)) + # if len(categorical_columns) == 0: + # assert len(choice_hyperparameter.value_range) == 1 + # if 'NoEmbedding' not in choice_hyperparameter.value_range: + # raise ValueError("Provided {} in choices, however, the dataset " + # "is incompatible with it".format(choice_hyperparameter.value_range)) + # embedding = CSH.CategoricalHyperparameter('__choice__', + # choice_hyperparameter.value_range, + # default_value=choice_hyperparameter.default_value) + # else: + + # if len(categorical_columns) == 0: + # default = 'NoEmbedding' + # if include is not None and default not in include: + # raise ValueError("Provided {} in include, however, the dataset " + # "is incompatible with it".format(include)) + # embedding = CSH.CategoricalHyperparameter('__choice__', + # ['NoEmbedding'], + # default_value=default) + # else: + # embedding = CSH.CategoricalHyperparameter('__choice__', + # list(available_embedding.keys()), + # default_value=default) + + # cs.add_hyperparameter(embedding) + # for name in embedding.choices: + # updates = self._get_search_space_updates(prefix=name) + # config_space = available_embedding[name].get_hyperparameter_search_space( + # dataset_properties, # type: ignore + # **updates) + # parent_hyperparameter = {'parent': embedding, 'value': name} + # cs.add_configuration_space( + # name, + # config_space, + # parent_hyperparameter=parent_hyperparameter + # ) + + # self.configuration_space_ = cs + # self.dataset_properties_ = dataset_properties + # return cs def transform(self, X: np.ndarray) -> np.ndarray: assert self.choice is not None, "Cannot call transform before the object is initialized" diff --git a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py index 3c01f75da..8f1d75040 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py +++ b/autoPyTorch/pipeline/components/setup/network_head/fully_connected.py @@ -82,7 +82,7 @@ def get_hyperparameter_search_space( log=units_layer.log, ) num_units_hp = get_hyperparameter(num_units_search_space, UniformIntegerHyperparameter) - + cs.add_hyperparameter(num_units_hp) if i >= min_num_layers and not num_layers_is_constant: # In the case of a constant, the max and min number of layers are the same. # So no condition is needed. If it is not a constant but a hyperparameter, diff --git a/autoPyTorch/pipeline/components/setup/network_head/no_head.py b/autoPyTorch/pipeline/components/setup/network_head/no_head.py index 0e711f06c..e95d25ffb 100644 --- a/autoPyTorch/pipeline/components/setup/network_head/no_head.py +++ b/autoPyTorch/pipeline/components/setup/network_head/no_head.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union from ConfigSpace.configuration_space import ConfigurationSpace from ConfigSpace.hyperparameters import CategoricalHyperparameter @@ -7,6 +7,7 @@ from torch import nn +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.network_head.base_network_head import NetworkHeadComponent from autoPyTorch.pipeline.components.setup.network_head.utils import _activations from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter @@ -28,7 +29,9 @@ def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...] return nn.Sequential(*layers) @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[str, Union[str, bool]]: + def get_properties( + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None + ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'NoHead', 'name': 'NoHead', @@ -39,7 +42,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[ @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict[str, str]] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, activation: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="activation", value_range=tuple(_activations.keys()), default_value=list(_activations.keys())[0]), diff --git a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py index 0dfc80707..113726870 100644 --- a/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py +++ b/autoPyTorch/pipeline/components/training/data_loader/base_data_loader.py @@ -115,7 +115,7 @@ def fit(self, X: Dict[str, Any], y: Any = None) -> torch.utils.data.DataLoader: shuffle=True, num_workers=X.get('num_workers', 0), pin_memory=X.get('pin_memory', True), - drop_last=X.get('drop_last', False), + drop_last=X.get('drop_last', True), collate_fn=custom_collate_fn, ) @@ -149,7 +149,6 @@ def get_loader(self, X: np.ndarray, y: Optional[np.ndarray] = None, batch_size: train_tensors=(X, y), seed=self.random_state.get_state()[1][0], # This dataset is used for loading test data in a batched format - seed=self.random_state.get_state()[1][0], shuffle=False, train_transforms=self.test_transform, val_transforms=self.test_transform, diff --git a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py index 0fefd9525..67ae71188 100644 --- a/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/AdversarialTrainer.py @@ -15,6 +15,7 @@ from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -91,7 +92,7 @@ def train_step(self, data: np.ndarray, targets: np.ndarray) -> Tuple[float, torc """ # prepare data = data.float().to(self.device) - targets = targets.long().to(self.device) + targets = self.cast_targets(targets) data, criterion_kwargs = self.data_preparation(data, targets) original_data = data[0] @@ -131,7 +132,7 @@ def fgsm_attack( """ data_copy = deepcopy(data) data_copy = data_copy.float().to(self.device) - targets = targets.long().to(self.device) + targets = self.cast_targets(targets) data_copy = torch.autograd.Variable(data_copy) data_copy.requires_grad = True @@ -146,7 +147,7 @@ def fgsm_attack( return adv_data @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { @@ -159,7 +160,7 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -240,7 +241,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py index 24346042d..9bf22f3b8 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutMixTrainer.py @@ -4,6 +4,7 @@ import torch +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp @@ -67,7 +68,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, BaseDatasetPropertiesType]] = None ) -> typing.Dict[str, typing.Union[str, bool]]: return { 'shortname': 'GridCutMixTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py index 4d7f1099d..fb6389fb8 100644 --- a/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/GridCutOutTrainer.py @@ -2,6 +2,7 @@ import numpy as np +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut @@ -52,7 +53,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y, 'y_b': y, 'lam': 1} @staticmethod - def get_properties(dataset_properties: typing.Optional[typing.Dict[str, typing.Any]] = None + def get_properties(dataset_properties: typing.Optional[typing.Dict[str, BaseDatasetPropertiesType]] = None ) -> typing.Dict[str, typing.Union[str, bool]]: return { 'shortname': 'GridCutOutTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py index 2bd015b46..1cd071ba6 100644 --- a/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/MixUpTrainer.py @@ -1,9 +1,10 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np import torch +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp @@ -43,7 +44,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return mixed_x, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'MixUpTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py index 3f7866f3c..bb4ccdb9a 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutMixTrainer.py @@ -1,9 +1,10 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np import torch +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.mixup_utils import MixUp @@ -56,7 +57,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'RowCutMixTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py index 4578082cb..7b679976e 100644 --- a/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/RowCutOutTrainer.py @@ -1,7 +1,8 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import numpy as np +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent from autoPyTorch.pipeline.components.training.trainer.cutout_utils import CutOut @@ -53,7 +54,7 @@ def data_preparation(self, X: np.ndarray, y: np.ndarray, return X, {'y_a': y_a, 'y_b': y_b, 'lam': lam} @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'RowCutOutTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py index fc5cc3e3a..c9202945c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/StandardTrainer.py @@ -1,13 +1,9 @@ -from typing import Callable, Dict, Optional, Tuple, Union - -from ConfigSpace.configuration_space import ConfigurationSpace -from ConfigSpace.hyperparameters import CategoricalHyperparameter +from typing import Any, Callable, Dict, Optional, Tuple, Union import numpy as np import torch -from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.base_trainer import BaseTrainerComponent @@ -60,7 +56,7 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam return lambda criterion, pred: criterion(pred, y_a) @staticmethod - def get_properties(dataset_properties: Optional[Dict[str, Any]] = None + def get_properties(dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None ) -> Dict[str, Union[str, bool]]: return { 'shortname': 'StandardTrainer', diff --git a/autoPyTorch/pipeline/components/training/trainer/__init__.py b/autoPyTorch/pipeline/components/training/trainer/__init__.py index ca09d7907..65e965bd2 100755 --- a/autoPyTorch/pipeline/components/training/trainer/__init__.py +++ b/autoPyTorch/pipeline/components/training/trainer/__init__.py @@ -129,7 +129,7 @@ def get_available_components( # Allow training schemes exclusive for some task types entry = available_comp[name] - task_type = dataset_properties['task_type'] + task_type = str(dataset_properties['task_type']) properties = entry.get_properties() if 'tabular' in task_type and not properties['handles_tabular']: continue @@ -284,9 +284,14 @@ def fit(self, X: Dict[str, Any], y: Any = None, **kwargs: Any) -> autoPyTorchCom **kwargs ) + # Comply with mypy + # Notice that choice here stands for the component choice framework, + # where we dynamically build the configuration space by selecting the available + # component choices. In this case, is what trainer choices are available + assert self.choice is not None + # Add snapshots to base network to enable # predicting with snapshot ensemble - self.choice: autoPyTorchComponent = cast(autoPyTorchComponent, self.choice) if self.choice.use_snapshot_ensemble: X['network_snapshots'].extend(self.choice.model_snapshots) return self.choice @@ -515,7 +520,6 @@ def early_stop_handler(self, X: Dict[str, Any]) -> bool: X (Dict[str, Any]): Dictionary with fitted parameters. It is a message passing mechanism, in which during a transform, a components adds relevant information so that further stages can be properly fitted - Returns: bool: If true, training should be stopped """ @@ -530,9 +534,13 @@ def early_stop_handler(self, X: Dict[str, Any]) -> bool: if self.checkpoint_dir is None: self.checkpoint_dir = tempfile.mkdtemp(dir=X['backend'].temporary_directory) +<<<<<<< HEAD last_epoch = self.run_summary.get_last_epoch() best_epoch = self.run_summary.get_best_epoch(split_type=self.early_stopping_split_type) epochs_since_best = last_epoch - best_epoch +======= + epochs_since_best = self.run_summary.get_last_epoch() - self.run_summary.get_best_epoch() +>>>>>>> [FIX] Tests after rebase of `reg_cocktails` (#359) # Save the checkpoint if there is a new best epoch best_path = os.path.join(self.checkpoint_dir, 'best.pth') diff --git a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py index fb4cbc77e..574b2002f 100644 --- a/autoPyTorch/pipeline/components/training/trainer/base_trainer.py +++ b/autoPyTorch/pipeline/components/training/trainer/base_trainer.py @@ -21,9 +21,9 @@ from torch.utils.tensorboard.writer import SummaryWriter -from autoPyTorch.constants import REGRESSION_TASKS +from autoPyTorch.constants import CLASSIFICATION_TASKS, REGRESSION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.setup.lr_scheduler.constants import StepIntervalUnit -from autoPyTorch.constants import REGRESSION_TASKS, CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES from autoPyTorch.pipeline.components.training.base_training import autoPyTorchTrainingComponent from autoPyTorch.pipeline.components.training.metrics.metrics import CLASSIFICATION_METRICS, REGRESSION_METRICS from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead @@ -593,7 +593,7 @@ def criterion_preparation(self, y_a: torch.Tensor, y_b: torch.Tensor = None, lam @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -663,7 +663,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py index 4feedf5cb..a181fe530 100644 --- a/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/cutout_utils.py @@ -13,6 +13,7 @@ from sklearn.utils import check_random_state from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -60,7 +61,7 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -146,7 +147,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py index e2ea25148..f9cd278a9 100644 --- a/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/mixup_utils.py @@ -13,6 +13,7 @@ from sklearn.utils import check_random_state from autoPyTorch.constants import CLASSIFICATION_TASKS, STRING_TO_TASK_TYPES +from autoPyTorch.datasets.base_dataset import BaseDatasetPropertiesType from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.utils.common import HyperparameterSearchSpace, add_hyperparameter, get_hyperparameter @@ -65,7 +66,7 @@ def criterion_preparation(self, y_a: np.ndarray, y_b: np.ndarray = None, lam: fl @staticmethod def get_hyperparameter_search_space( - dataset_properties: Optional[Dict] = None, + dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None, weighted_loss: HyperparameterSearchSpace = HyperparameterSearchSpace( hyperparameter="weighted_loss", value_range=(1, ), @@ -145,7 +146,7 @@ def get_hyperparameter_search_space( # remove the code below. Also update the method signature, so the weighted loss # is not a constant. if dataset_properties is not None: - if STRING_TO_TASK_TYPES[dataset_properties['task_type']] in CLASSIFICATION_TASKS: + if STRING_TO_TASK_TYPES[str(dataset_properties['task_type'])] in CLASSIFICATION_TASKS: add_hyperparameter(cs, weighted_loss, Constant) return cs diff --git a/autoPyTorch/pipeline/components/training/trainer/utils.py b/autoPyTorch/pipeline/components/training/trainer/utils.py index cdc22402f..ce16d5e3c 100644 --- a/autoPyTorch/pipeline/components/training/trainer/utils.py +++ b/autoPyTorch/pipeline/components/training/trainer/utils.py @@ -105,7 +105,7 @@ def get_la_step(self) -> int: return self._la_step def state_dict(self) -> Dict[str, Any]: - return self.optimizer.state_dict() + return self.optimizer.state_dict() # type: ignore[no-any-return] def load_state_dict(self, state_dict: Dict[str, Any]) -> None: self.optimizer.load_state_dict(state_dict) @@ -129,7 +129,7 @@ def _clear_and_load_backup(self) -> None: @property def param_groups(self) -> List[Dict]: - return self.optimizer.param_groups + return self.optimizer.param_groups # type: ignore[no-any-return] def step(self, closure: Optional[Callable] = None) -> torch.Tensor: """Performs a single Lookahead optimization step. diff --git a/autoPyTorch/pipeline/tabular_classification.py b/autoPyTorch/pipeline/tabular_classification.py index bc9616f58..720d0af64 100644 --- a/autoPyTorch/pipeline/tabular_classification.py +++ b/autoPyTorch/pipeline/tabular_classification.py @@ -142,23 +142,10 @@ def _predict_proba(self, X: np.ndarray) -> np.ndarray: # The final layer is always softmax now (`pred` already gives pseudo proba) return pred else: -<<<<<<< HEAD raise ValueError("Expected output_shape to be integer, got {}," "Tabular Classification only supports 'binary' and 'multiclass' outputs" "got {}".format(type(self.dataset_properties['output_shape']), self.dataset_properties['output_type'])) -======= - all_proba = [] - - for k in range(self.dataset_properties['output_shape']): - proba_k = pred[:, k, :self.dataset_properties['output_shape'][k]] - normalizer = proba_k.sum(axis=1)[:, np.newaxis] - normalizer[normalizer == 0.0] = 1.0 - proba_k /= normalizer - all_proba.append(proba_k) - - return np.array(all_proba) ->>>>>>> Bug fixes (#249) def predict_proba(self, X: np.ndarray, batch_size: Optional[int] = None) -> np.ndarray: """predict_proba. diff --git a/autoPyTorch/utils/backend.py b/autoPyTorch/utils/backend.py deleted file mode 100644 index 5348bd11c..000000000 --- a/autoPyTorch/utils/backend.py +++ /dev/null @@ -1,575 +0,0 @@ -import glob -import os -import pickle -import re -import shutil -import tempfile -import time -import uuid -import warnings -from typing import Dict, List, Optional, Tuple, Union - -import lockfile - -import numpy as np - -from autoPyTorch.datasets.base_dataset import BaseDataset -from autoPyTorch.ensemble.abstract_ensemble import AbstractEnsemble -from autoPyTorch.pipeline.base_pipeline import BasePipeline -from autoPyTorch.utils.logging_ import PicklableClientLogger, get_named_client_logger - -__all__ = [ - 'Backend' -] - - -def create( - temporary_directory: Optional[str], - output_directory: Optional[str], - delete_tmp_folder_after_terminate: bool = True, - delete_output_folder_after_terminate: bool = True, -) -> 'Backend': - """ - Creates a backend object that manages disk related transactions - - Args: - temporary_directory (str): where all temporal data is to be dumped - output_directory (str): where all predictions are to be output - delete_tmp_folder_after_terminate (bool): whether to delete the - temporal directory when then run completes - delete_output_folder_after_terminate (bool): whether to delete - the output directory when the run completes - - Returns: - Backend object - """ - context = BackendContext(temporary_directory, output_directory, - delete_tmp_folder_after_terminate, - delete_output_folder_after_terminate, - ) - backend = Backend(context) - - return backend - - -def get_randomized_directory_name(temporary_directory: Optional[str] = None) -> str: - uuid_str = str(uuid.uuid1(clock_seq=os.getpid())) - - temporary_directory = ( - temporary_directory - if temporary_directory - else os.path.join( - tempfile.gettempdir(), - "autoPyTorch_tmp_{}".format( - uuid_str, - ), - ) - ) - - return temporary_directory - - -class BackendContext(object): - - def __init__(self, - temporary_directory: Optional[str], - output_directory: Optional[str], - delete_tmp_folder_after_terminate: bool, - delete_output_folder_after_terminate: bool, - ): - - # Check that the names of tmp_dir and output_dir is not the same. - if temporary_directory == output_directory and temporary_directory is not None: - raise ValueError("The temporary and the output directory " - "must be different.") - - self.delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate - self.delete_output_folder_after_terminate = delete_output_folder_after_terminate - # attributes to check that directories were created by autoPyTorch - self._tmp_dir_created = False - self._output_dir_created = False - - self._temporary_directory = ( - get_randomized_directory_name( - temporary_directory=temporary_directory, - ) - ) - self._output_directory = output_directory - self.create_directories() - self._logger = None # type: Optional[PicklableClientLogger] - - @property - def output_directory(self) -> Optional[str]: - if self._output_directory is not None: - # make sure that tilde does not appear on the path. - return os.path.expanduser(os.path.expandvars(self._output_directory)) - else: - return None - - @property - def temporary_directory(self) -> str: - # make sure that tilde does not appear on the path. - return os.path.expanduser(os.path.expandvars(self._temporary_directory)) - - def create_directories(self) -> None: - # Exception is raised if self.temporary_directory already exists. - os.makedirs(self.temporary_directory) - self._tmp_dir_created = True - - # Exception is raised if self.output_directory already exists. - if self.output_directory is not None: - os.makedirs(self.output_directory) - self._output_dir_created = True - - def delete_directories(self, force: bool = True) -> None: - if self.output_directory and (self.delete_output_folder_after_terminate or force): - if self._output_dir_created is False: - raise ValueError("Failed to delete output dir: %s because autoPyTorch did not " - "create it. Please make sure that the specified output dir does " - "not exist when instantiating autoPyTorch." - % self.output_directory) - try: - shutil.rmtree(self.output_directory) - except Exception: - try: - if self._logger is not None: - self._logger.warning("Could not delete output dir: %s" % - self.output_directory) - else: - warnings.warn("Could not delete output dir: %s" % self.output_directory) - except Exception: - warnings.warn("Could not delete output dir: %s" % self.output_directory) - - if self.delete_tmp_folder_after_terminate or force: - if self._tmp_dir_created is False: - raise ValueError("Failed to delete tmp dir: % s because autoPyTorch did not " - "create it. Please make sure that the specified tmp dir does not " - "exist when instantiating autoPyTorch." - % self.temporary_directory) - try: - shutil.rmtree(self.temporary_directory) - except Exception: - try: - if self._logger is not None: - self._logger.warning( - "Could not delete tmp dir: %s" % self.temporary_directory) - else: - warnings.warn("Could not delete tmp dir: %s" % self.temporary_directory) - except Exception: - warnings.warn("Could not delete tmp dir: %s" % self.temporary_directory) - - -class Backend(object): - """Utility class to load and save all objects to be persisted. - These are: - * start time of auto-pytorch - * true targets of the ensemble - """ - - def __init__(self, context: BackendContext): - self._logger = None # type: Optional[PicklableClientLogger] - self.context = context - - # Track the number of configurations launched - # num_run == 1 means a dummy estimator run - self.active_num_run = 1 - - # Create the temporary directory if it does not yet exist - try: - os.makedirs(self.temporary_directory) - except Exception: - pass - # This does not have to exist or be specified - if self.output_directory is not None: - if not os.path.exists(self.output_directory): - raise ValueError("Output directory %s does not exist." % self.output_directory) - - self.internals_directory = os.path.join(self.temporary_directory, ".autoPyTorch") - self._make_internals_directory() - - def setup_logger(self, name: str, port: int) -> None: - self._logger = get_named_client_logger( - name=name, - port=port, - ) - self.context._logger = self._logger - return - - @property - def output_directory(self) -> Optional[str]: - return self.context.output_directory - - @property - def temporary_directory(self) -> str: - return self.context.temporary_directory - - def _make_internals_directory(self) -> None: - # TODO: make exist_ok a function argument - try: - os.makedirs(self.internals_directory, exist_ok=True) - except Exception as e: - if self._logger is not None: - self._logger.debug("_make_internals_directory: %s" % e) - try: - os.makedirs(self.get_runs_directory(), exist_ok=True) - except Exception as e: - if self._logger is not None: - self._logger.debug("_make_internals_directory: %s" % e) - - def _get_start_time_filename(self, seed: Union[str, int]) -> str: - if isinstance(seed, str): - seed = int(seed) - return os.path.join(self.internals_directory, "start_time_%d" % seed) - - def save_start_time(self, seed: str) -> str: - self._make_internals_directory() - start_time = time.time() - - filepath = self._get_start_time_filename(seed) - - if not isinstance(start_time, float): - raise ValueError("Start time must be a float, but is %s." % type(start_time)) - - if os.path.exists(filepath): - raise ValueError( - "{filepath} already exist. Different seeds should be provided for different jobs." - ) - - with tempfile.NamedTemporaryFile('w', dir=os.path.dirname(filepath), delete=False) as fh: - fh.write(str(start_time)) - tempname = fh.name - os.rename(tempname, filepath) - - return filepath - - def load_start_time(self, seed: int) -> float: - with open(self._get_start_time_filename(seed), 'r') as fh: - start_time = float(fh.read()) - return start_time - - def get_smac_output_directory(self) -> str: - return os.path.join(self.temporary_directory, 'smac3-output') - - def get_smac_output_directory_for_run(self, seed: int) -> str: - return os.path.join( - self.temporary_directory, - 'smac3-output', - 'run_%d' % seed - ) - - def _get_targets_ensemble_filename(self) -> str: - return os.path.join(self.internals_directory, - "true_targets_ensemble.npy") - - def save_targets_ensemble(self, targets: np.ndarray) -> str: - self._make_internals_directory() - if not isinstance(targets, np.ndarray): - raise ValueError('Targets must be of type np.ndarray, but is %s' % - type(targets)) - - filepath = self._get_targets_ensemble_filename() - - # Try to open the file without locking it, this will reduce the - # number of times where we erroneously keep a lock on the ensemble - # targets file although the process already was killed - try: - existing_targets = np.load(filepath, allow_pickle=True) - if existing_targets.shape[0] > targets.shape[0] or ( - existing_targets.shape == targets.shape and np.allclose(existing_targets, targets)): - return filepath - except Exception: - pass - - with lockfile.LockFile(filepath): - if os.path.exists(filepath): - with open(filepath, 'rb') as fh: - existing_targets = np.load(fh, allow_pickle=True) - if existing_targets.shape[0] > targets.shape[0] or ( - existing_targets.shape == targets.shape and np.allclose(existing_targets, targets)): - return filepath - - with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( - filepath), delete=False) as fh_w: - np.save(fh_w, targets.astype(np.float32)) - tempname = fh_w.name - - os.rename(tempname, filepath) - - return filepath - - def load_targets_ensemble(self) -> np.ndarray: - filepath = self._get_targets_ensemble_filename() - - with lockfile.LockFile(filepath): - with open(filepath, 'rb') as fh: - targets = np.load(fh, allow_pickle=True) - - return targets - - def _get_datamanager_pickle_filename(self) -> str: - return os.path.join(self.internals_directory, 'datamanager.pkl') - - def save_datamanager(self, datamanager: BaseDataset) -> str: - self._make_internals_directory() - filepath = self._get_datamanager_pickle_filename() - - with lockfile.LockFile(filepath): - if not os.path.exists(filepath): - with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( - filepath), delete=False) as fh: - pickle.dump(datamanager, fh, -1) - tempname = fh.name - os.rename(tempname, filepath) - - return filepath - - def load_datamanager(self) -> BaseDataset: - filepath = self._get_datamanager_pickle_filename() - with lockfile.LockFile(filepath): - with open(filepath, 'rb') as fh: - return pickle.load(fh) - - def replace_datamanager(self, datamanager: BaseDataset) -> None: - """ - This function is called to replace the old datamanager with a datamanager - in case it is required. - - Args: - datamanager (BaseDataset): the new datamanager to replace the old. - """ - warnings.warn("Original dataset will be overwritten with the provided dataset") - datamanager_pickle_file = self._get_datamanager_pickle_filename() - if os.path.exists(datamanager_pickle_file): - os.remove(datamanager_pickle_file) - self.save_datamanager(datamanager=datamanager) - - def get_runs_directory(self) -> str: - return os.path.join(self.internals_directory, 'runs') - - def get_numrun_directory(self, seed: int, num_run: int, budget: float) -> str: - return os.path.join(self.internals_directory, 'runs', '%d_%d_%s' % (seed, num_run, budget)) - - def get_next_num_run(self, peek: bool = False) -> int: - """ - Every pipeline that is fitted by the estimator is stored with an - identifier called num_run. A dummy classifier will always have a num_run - equal to 1, and all other new configurations that are explored will - have a sequentially increasing identifier. - - This method returns the next num_run a configuration should take. - - Parameters - ---------- - peek: bool - By default, the next num_rum will be returned, i.e. self.active_num_run + 1 - Yet, if this bool parameter is equal to True, the value of the current - num_run is provided, i.e, self.active_num_run. - In other words, peek allows to get the current maximum identifier - of a configuration. - - Returns - ------- - num_run: int - An unique identifier for a configuration - """ - - # If there are other num_runs, their name would be runs/__ - other_num_runs = [int(os.path.basename(run_dir).split('_')[1]) - for run_dir in glob.glob(os.path.join(self.internals_directory, 'runs', '*')) - if re.match(r"\d+_\d+_\d+", os.path.basename(run_dir))] - if len(other_num_runs) > 0: - # We track the number of runs from two forefronts: - # The physically available num_runs (which might be deleted or a crash could happen) - # From a internally kept attribute. The later should be sufficient, but we - # want to be robust against multiple backend copies on different workers - self.active_num_run = max([self.active_num_run] + other_num_runs) - - # We are interested in the next run id - if not peek: - self.active_num_run += 1 - return self.active_num_run - - def get_model_filename(self, seed: int, idx: int, budget: float) -> str: - return '%s.%s.%s.model' % (seed, idx, budget) - - def get_cv_model_filename(self, seed: int, idx: int, budget: float) -> str: - return '%s.%s.%s.cv_model' % (seed, idx, budget) - - def list_all_models(self, seed: int) -> List[str]: - runs_directory = self.get_runs_directory() - model_files = glob.glob( - os.path.join(glob.escape(runs_directory), '%d_*' % seed, '%s.*.*.model' % seed) - ) - return model_files - - def load_models_by_identifiers(self, identifiers: List[Tuple[int, int, float]] - ) -> Dict: - models = dict() - - for identifier in identifiers: - seed, idx, budget = identifier - models[identifier] = self.load_model_by_seed_and_id_and_budget( - seed, idx, budget) - - return models - - def load_model_by_seed_and_id_and_budget(self, seed: int, - idx: int, - budget: float - ) -> BasePipeline: - model_directory = self.get_numrun_directory(seed, idx, budget) - - model_file_name = '%s.%s.%s.model' % (seed, idx, budget) - model_file_path = os.path.join(model_directory, model_file_name) - with open(model_file_path, 'rb') as fh: - return pickle.load(fh) - - def load_cv_models_by_identifiers(self, identifiers: List[Tuple[int, int, float]] - ) -> Dict: - models = dict() - - for identifier in identifiers: - seed, idx, budget = identifier - models[identifier] = self.load_cv_model_by_seed_and_id_and_budget( - seed, idx, budget) - - return models - - def load_cv_model_by_seed_and_id_and_budget(self, - seed: int, - idx: int, - budget: float - ) -> BasePipeline: - model_directory = self.get_numrun_directory(seed, idx, budget) - - model_file_name = '%s.%s.%s.cv_model' % (seed, idx, budget) - model_file_path = os.path.join(model_directory, model_file_name) - with open(model_file_path, 'rb') as fh: - return pickle.load(fh) - - def save_numrun_to_dir( - self, seed: int, idx: int, budget: float, model: Optional[BasePipeline], - cv_model: Optional[BasePipeline], ensemble_predictions: Optional[np.ndarray], - valid_predictions: Optional[np.ndarray], test_predictions: Optional[np.ndarray], - ) -> None: - assert self._logger is not None - runs_directory = self.get_runs_directory() - tmpdir = tempfile.mkdtemp(dir=runs_directory) - if model is not None: - file_path = os.path.join(tmpdir, self.get_model_filename(seed, idx, budget)) - with open(file_path, 'wb') as fh: - pickle.dump(model, fh, -1) - - if cv_model is not None: - file_path = os.path.join(tmpdir, self.get_cv_model_filename(seed, idx, budget)) - with open(file_path, 'wb') as fh: - pickle.dump(cv_model, fh, -1) - - for preds, subset in ( - (ensemble_predictions, 'ensemble'), - (valid_predictions, 'valid'), - (test_predictions, 'test') - ): - if preds is not None: - file_path = os.path.join( - tmpdir, - self.get_prediction_filename(subset, seed, idx, budget) - ) - with open(file_path, 'wb') as fh: - pickle.dump(preds.astype(np.float32), fh, -1) - try: - self._logger.debug("Renaming {} to {}".format(tmpdir, - self.get_numrun_directory(seed, idx, budget))) - os.rename(tmpdir, self.get_numrun_directory(seed, idx, budget)) - except OSError: - if os.path.exists(self.get_numrun_directory(seed, idx, budget)): - os.rename(self.get_numrun_directory(seed, idx, budget), - os.path.join(runs_directory, tmpdir + '.old')) - os.rename(tmpdir, self.get_numrun_directory(seed, idx, budget)) - shutil.rmtree(os.path.join(runs_directory, tmpdir + '.old')) - - def get_ensemble_dir(self) -> str: - return os.path.join(self.internals_directory, 'ensembles') - - def load_ensemble(self, seed: int) -> Optional[AbstractEnsemble]: - ensemble_dir = self.get_ensemble_dir() - - if not os.path.exists(ensemble_dir): - if self._logger is not None: - self._logger.warning('Directory %s does not exist' % ensemble_dir) - else: - warnings.warn('Directory %s does not exist' % ensemble_dir) - return None - - if seed >= 0: - indices_files = glob.glob( - os.path.join(glob.escape(ensemble_dir), '%s.*.ensemble' % seed) - ) - indices_files.sort() - else: - indices_files = os.listdir(ensemble_dir) - indices_files = [os.path.join(ensemble_dir, f) for f in indices_files] - indices_files.sort(key=lambda f: time.ctime(os.path.getmtime(f))) - - with open(indices_files[-1], 'rb') as fh: - ensemble_members_run_numbers = pickle.load(fh) - - return ensemble_members_run_numbers - - def save_ensemble(self, ensemble: AbstractEnsemble, idx: int, seed: int) -> None: - try: - os.makedirs(self.get_ensemble_dir()) - except Exception: - pass - - filepath = os.path.join( - self.get_ensemble_dir(), - '%s.%s.ensemble' % (str(seed), str(idx).zfill(10)) - ) - with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname( - filepath), delete=False) as fh: - pickle.dump(ensemble, fh) - tempname = fh.name - os.rename(tempname, filepath) - - def get_prediction_filename(self, subset: str, - automl_seed: Union[str, int], - idx: int, - budget: float - ) -> str: - return 'predictions_%s_%s_%s_%s.npy' % (subset, automl_seed, idx, budget) - - def save_predictions_as_txt(self, - predictions: np.ndarray, - subset: str, - idx: int, precision: int, - prefix: Optional[str] = None) -> None: - if not self.output_directory: - return - # Write prediction scores in prescribed format - filepath = os.path.join( - self.output_directory, - ('%s_' % prefix if prefix else '') + '%s_%s.predict' % (subset, str(idx)), - ) - - format_string = '{:.%dg} ' % precision - with tempfile.NamedTemporaryFile('w', dir=os.path.dirname( - filepath), delete=False) as output_file: - for row in predictions: - if not isinstance(row, np.ndarray) and not isinstance(row, list): - row = [row] - for val in row: - output_file.write(format_string.format(float(val))) - output_file.write('\n') - tempname = output_file.name - os.rename(tempname, filepath) - - def write_txt_file(self, filepath: str, data: str, name: str) -> None: - with lockfile.LockFile(filepath): - with tempfile.NamedTemporaryFile('w', dir=os.path.dirname( - filepath), delete=False) as fh: - fh.write(data) - tempname = fh.name - os.rename(tempname, filepath) - if self._logger is not None: - self._logger.debug('Created %s file %s' % (name, filepath)) diff --git a/examples/40_advanced/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/40_advanced/example_custom_configuration_space.py deleted file mode 100644 index 25eb86be7..000000000 --- a/examples/40_advanced/40_advanced/example_custom_configuration_space.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -====================== -Tabular Classification with Custom Configuration Space -====================== - -The following example shows how adjust the configuration space of -the search. Currently, there are two changes that can be made to the space:- -1. Adjust individual hyperparameters in the pipeline -2. Include or exclude components: - a) include: Dictionary containing components to include. Key is the node - name and Value is an Iterable of the names of the components - to include. Only these components will be present in the - search space. - b) exclude: Dictionary containing components to exclude. Key is the node - name and Value is an Iterable of the names of the components - to exclude. All except these components will be present in - the search space. -""" -import os -import tempfile as tmp -import warnings - -os.environ['JOBLIB_TEMP_FOLDER'] = tmp.gettempdir() -os.environ['OMP_NUM_THREADS'] = '1' -os.environ['OPENBLAS_NUM_THREADS'] = '1' -os.environ['MKL_NUM_THREADS'] = '1' - -warnings.simplefilter(action='ignore', category=UserWarning) -warnings.simplefilter(action='ignore', category=FutureWarning) - -import sklearn.datasets -import sklearn.model_selection - -from autoPyTorch.api.tabular_classification import TabularClassificationTask -from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates - - -def get_search_space_updates(): - """ - Search space updates to the task can be added using HyperparameterSearchSpaceUpdates - Returns: - HyperparameterSearchSpaceUpdates - """ - updates = HyperparameterSearchSpaceUpdates() - updates.append(node_name="data_loader", - hyperparameter="batch_size", - value_range=[16, 512], - default_value=32) - updates.append(node_name="lr_scheduler", - hyperparameter="CosineAnnealingLR:T_max", - value_range=[50, 60], - default_value=55) - updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:dropout', - value_range=[0, 0.5], - default_value=0.2) - updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:multi_branch_choice', - value_range=['shake-shake'], - default_value='shake-shake') - updates.append(node_name='network_backbone', - hyperparameter='ResNetBackbone:shake_shake_update_func', - value_range=['M3'], - default_value='M3' - ) - return updates - - -if __name__ == '__main__': - - ############################################################################ - # Data Loading - # ============ - X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, - ) - - ############################################################################ - # Build and fit a classifier with include components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['ResNetBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - print(api.run_history, api.trajectory) - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) - - ############################################################################ - # Build and fit a classifier with exclude components - # ================================================== - api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} - ) - - ############################################################################ - # Search for an ensemble of machine learning algorithms - # ===================================================== - api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=300, - func_eval_time_limit_secs=50 - ) - - ############################################################################ - # Print the final ensemble performance - # ==================================== - print(api.run_history, api.trajectory) - y_pred = api.predict(X_test) - score = api.score(y_pred, y_test) - print(score) - print(api.show_models()) diff --git a/examples/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/example_custom_configuration_space.py index 985d9d9ff..25eb86be7 100644 --- a/examples/40_advanced/example_custom_configuration_space.py +++ b/examples/40_advanced/example_custom_configuration_space.py @@ -5,7 +5,6 @@ The following example shows how adjust the configuration space of the search. Currently, there are two changes that can be made to the space:- - 1. Adjust individual hyperparameters in the pipeline 2. Include or exclude components: a) include: Dictionary containing components to include. Key is the node @@ -55,81 +54,88 @@ def get_search_space_updates(): hyperparameter='ResNetBackbone:dropout', value_range=[0, 0.5], default_value=0.2) + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:multi_branch_choice', + value_range=['shake-shake'], + default_value='shake-shake') + updates.append(node_name='network_backbone', + hyperparameter='ResNetBackbone:shake_shake_update_func', + value_range=['M3'], + default_value='M3' + ) return updates -############################################################################ -# Data Loading -# ============ -X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) -X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, - y, - random_state=1, -) - -############################################################################ -# Build and fit a classifier with include components -# ================================================== -api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - include_components={'network_backbone': ['MLPBackbone', 'ResNetBackbone'], - 'encoder': ['OneHotEncoder']} -) - -############################################################################ -# Search for an ensemble of machine learning algorithms -# ===================================================== -api.search( - X_train=X_train.copy(), - y_train=y_train.copy(), - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 -) - -############################################################################ -# Print the final ensemble performance -# ==================================== -y_pred = api.predict(X_test) -score = api.score(y_pred, y_test) -print(score) -print(api.show_models()) - -# Print statistics from search -print(api.sprint_statistics()) - -############################################################################ -# Build and fit a classifier with exclude components -# ================================================== -api = TabularClassificationTask( - search_space_updates=get_search_space_updates(), - exclude_components={'network_backbone': ['MLPBackbone'], - 'encoder': ['OneHotEncoder']} -) - -############################################################################ -# Search for an ensemble of machine learning algorithms -# ===================================================== -api.search( - X_train=X_train, - y_train=y_train, - X_test=X_test.copy(), - y_test=y_test.copy(), - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=30 -) - -############################################################################ -# Print the final ensemble performance -# ==================================== -y_pred = api.predict(X_test) -score = api.score(y_pred, y_test) -print(score) -print(api.show_models()) - -# Print statistics from search -print(api.sprint_statistics()) +if __name__ == '__main__': + + ############################################################################ + # Data Loading + # ============ + X, y = sklearn.datasets.fetch_openml(data_id=40981, return_X_y=True, as_frame=True) + X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( + X, + y, + random_state=1, + ) + + ############################################################################ + # Build and fit a classifier with include components + # ================================================== + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + include_components={'network_backbone': ['ResNetBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train.copy(), + y_train=y_train.copy(), + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) + + ############################################################################ + # Build and fit a classifier with exclude components + # ================================================== + api = TabularClassificationTask( + search_space_updates=get_search_space_updates(), + exclude_components={'network_backbone': ['MLPBackbone'], + 'encoder': ['OneHotEncoder']} + ) + + ############################################################################ + # Search for an ensemble of machine learning algorithms + # ===================================================== + api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test.copy(), + y_test=y_test.copy(), + optimize_metric='accuracy', + total_walltime_limit=300, + func_eval_time_limit_secs=50 + ) + + ############################################################################ + # Print the final ensemble performance + # ==================================== + print(api.run_history, api.trajectory) + y_pred = api.predict(X_test) + score = api.score(y_pred, y_test) + print(score) + print(api.show_models()) diff --git a/examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py b/examples/40_advanced/example_posthoc_ensemble_fit.py similarity index 100% rename from examples/40_advanced/40_advanced/example_posthoc_ensemble_fit.py rename to examples/40_advanced/example_posthoc_ensemble_fit.py diff --git a/requirements.txt b/requirements.txt index 5582e1793..1f2dd38b6 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ distributed>=2.2.0 catboost lightgbm flaky -tabulate +tabulate \ No newline at end of file diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 49c77159d..010342f59 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -151,7 +151,6 @@ def test_tabular_classification(openml_id, resampling_strategy, backend, resampl run_key_model_run_dir, f"{estimator.seed}.{successful_num_run}.{run_key.budget}.cv_model" ) - time.sleep(5) assert os.path.exists(model_file), print_debug_information(estimator) model = estimator._backend.load_cv_model_by_seed_and_id_and_budget( diff --git a/test/test_data/test_feature_validator.py b/test/test_data/test_feature_validator.py index a851f092e..89d59e5b4 100644 --- a/test/test_data/test_feature_validator.py +++ b/test/test_data/test_feature_validator.py @@ -465,9 +465,4 @@ def test_feature_validator_imbalanced_data(): transformed_X_test = validator.transform(X_test) transformed_X_test = pd.DataFrame(transformed_X_test) - null_columns = [] - for column in transformed_X_test.columns: - if transformed_X_test[column].isna().all(): - null_columns.append(column) - - assert null_columns == [1] + assert not len(validator.all_nan_columns) diff --git a/test/test_evaluation/test_fit_evaluator.py b/test/test_evaluation/test_fit_evaluator.py index 4e760a50c..1515ba74f 100644 --- a/test/test_evaluation/test_fit_evaluator.py +++ b/test/test_evaluation/test_fit_evaluator.py @@ -14,12 +14,12 @@ from smac.tae import StatusType +from autoPyTorch.automl_common.common.utils.backend import create from autoPyTorch.datasets.resampling_strategy import NoResamplingStrategyTypes from autoPyTorch.evaluation.fit_evaluator import FitEvaluator from autoPyTorch.evaluation.utils import read_queue from autoPyTorch.pipeline.base_pipeline import BasePipeline from autoPyTorch.pipeline.components.training.metrics.metrics import accuracy -from autoPyTorch.utils import backend this_directory = os.path.dirname(__file__) sys.path.append(this_directory) @@ -93,9 +93,10 @@ def test_no_resampling(self, pipeline_mock): lambda X, batch_size=None: np.tile([0.6, 0.4], (len(X), 1)) pipeline_mock.side_effect = lambda **kwargs: pipeline_mock pipeline_mock.get_additional_run_info.return_value = None + pipeline_mock.get_default_pipeline_options.return_value = {'budget_type': 'epochs', 'epochs': 10} configuration = unittest.mock.Mock(spec=Configuration) - backend_api = backend.create(self.tmp_dir, self.output_dir) + backend_api = create(self.tmp_dir, self.output_dir, 'autoPyTorch') backend_api.load_datamanager = lambda: D queue_ = multiprocessing.Queue() @@ -182,7 +183,7 @@ def test_predict_proba_binary_classification(self, mock): [[0.1, 0.9]] * y.shape[0] ) mock.side_effect = lambda **kwargs: mock - + mock.get_default_pipeline_options.return_value = {'budget_type': 'epochs', 'epochs': 10} configuration = unittest.mock.Mock(spec=Configuration) queue_ = multiprocessing.Queue() diff --git a/test/test_pipeline/components/setup/test_setup.py b/test/test_pipeline/components/setup/test_setup.py index 0b618e504..99f33d81f 100644 --- a/test/test_pipeline/components/setup/test_setup.py +++ b/test/test_pipeline/components/setup/test_setup.py @@ -445,11 +445,11 @@ def test_add_network_backbone(self): # clear addons base_network_backbone_choice._addons = ThirdPartyComponents(NetworkBackboneComponent) - @pytest.mark.parametrize('resnet_shape', ['funnel', 'long_funnel', - 'diamond', 'hexagon', - 'brick', 'triangle', - 'stairs']) - def test_dropout(self, resnet_shape): + @pytest.mark.parametrize('dropout_shape', ['funnel', 'long_funnel', + 'diamond', 'hexagon', + 'brick', 'triangle', + 'stairs']) + def test_dropout(self, dropout_shape): # ensures that dropout is assigned to the resblock as expected dataset_properties = {"task_type": constants.TASK_TYPES_TO_STRING[1]} max_dropout = 0.5 @@ -463,10 +463,10 @@ def test_dropout(self, resnet_shape): hyperparameter='max_dropout', value_range=[max_dropout], default_value=max_dropout), - resnet_shape=HyperparameterSearchSpace( - hyperparameter='resnet_shape', - value_range=[resnet_shape], - default_value=resnet_shape), + dropout_shape=HyperparameterSearchSpace( + hyperparameter='dropout_shape', + value_range=[dropout_shape], + default_value=dropout_shape), num_groups=HyperparameterSearchSpace( hyperparameter='num_groups', value_range=[num_groups], @@ -481,9 +481,10 @@ def test_dropout(self, resnet_shape): config = config_space.sample_configuration().get_dictionary() resnet_backbone = ShapedResNetBackbone(**config) backbone = resnet_backbone.build_backbone((100, 5)) - dropout_probabilites = [resnet_backbone.config[key] for key in resnet_backbone.config if 'dropout_' in key] + dropout_probabilites = [resnet_backbone.config[key] for key in resnet_backbone.config + if 'dropout_' in key and 'shape' not in key] dropout_shape = get_shaped_neuron_counts( - shape=resnet_shape, + shape=dropout_shape, in_feat=0, out_feat=0, max_neurons=max_dropout, diff --git a/test/test_pipeline/components/training/test_training.py b/test/test_pipeline/components/training/test_training.py index 44a903308..39fa7668e 100644 --- a/test/test_pipeline/components/training/test_training.py +++ b/test/test_pipeline/components/training/test_training.py @@ -21,6 +21,7 @@ ) from autoPyTorch.pipeline.components.training.trainer import ( TrainerChoice, +) from autoPyTorch.pipeline.components.training.trainer.AdversarialTrainer import ( AdversarialTrainer ) @@ -352,6 +353,7 @@ def test_classification_epoch_training(self, n_samples): if counter > epochs: pytest.fail(f"Could not overfit a dummy classification under {epochs} epochs") + def test_every_trainer_is_valid(): """ Makes sure that every trainer is a valid estimator. @@ -517,7 +519,7 @@ def dummy_performance(*args, **kwargs): 'step_interval': StepIntervalUnit.batch } for item in ['backend', 'lr_scheduler', 'network', 'optimizer', 'train_data_loader', 'val_data_loader', - 'device', 'y_train']: + 'device', 'y_train', 'network_snapshots']: fit_dictionary[item] = unittest.mock.MagicMock() fit_dictionary['backend'].temporary_directory = tempfile.mkdtemp() @@ -537,9 +539,9 @@ def dummy_performance(*args, **kwargs): shutil.rmtree(fit_dictionary['backend'].temporary_directory) -class AdversarialTrainerTest(BaseTraining, unittest.TestCase): +class TestAdversarialTrainer(BaseTraining): - def test_epoch_training(self): + def test_epoch_training(self, n_samples): """ Makes sure we are able to train a model and produce good training performance @@ -550,8 +552,10 @@ def test_epoch_training(self): loader, _, epochs, - logger) = self.prepare_trainer(AdversarialTrainer(epsilon=0.07), - constants.TABULAR_CLASSIFICATION) + logger) = self.prepare_trainer(n_samples, + AdversarialTrainer(epsilon=0.07), + constants.TABULAR_CLASSIFICATION, + OVERFIT_EPOCHS) # Train the model counter = 0 diff --git a/test/test_pipeline/test_tabular_classification.py b/test/test_pipeline/test_tabular_classification.py index 4ed5f0f32..f557bd855 100644 --- a/test/test_pipeline/test_tabular_classification.py +++ b/test/test_pipeline/test_tabular_classification.py @@ -25,8 +25,10 @@ from autoPyTorch.pipeline.components.training.trainer.utils import Lookahead from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline from autoPyTorch.utils.common import FitRequirement -from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates, \ +from autoPyTorch.utils.hyperparameter_search_space_update import ( + HyperparameterSearchSpaceUpdates, parse_hyperparameter_search_space_updates +) @pytest.fixture @@ -570,6 +572,12 @@ def test_train_pipeline_with_runtime(fit_dictionary_tabular_dummy): cs = pipeline.get_hyperparameter_search_space() config = cs.get_default_configuration() + trainer = config.get('trainer:__choice__') + config_dict = config.get_dictionary() + config_dict[f'trainer:{trainer}:use_stochastic_weight_averaging'] = False + config_dict[f'trainer:{trainer}:use_snapshot_ensemble'] = False + del config_dict[f'trainer:{trainer}:se_lastk'] + config = Configuration(cs, values=config_dict) pipeline.set_hyperparameters(config) pipeline.fit(fit_dictionary_tabular_dummy) diff --git a/test/test_pipeline/test_tabular_regression.py b/test/test_pipeline/test_tabular_regression.py index 5889ed1c6..a2c3b695e 100644 --- a/test/test_pipeline/test_tabular_regression.py +++ b/test/test_pipeline/test_tabular_regression.py @@ -20,6 +20,7 @@ from autoPyTorch.pipeline.tabular_regression import TabularRegressionPipeline from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.hyperparameter_search_space_update import ( + HyperparameterSearchSpaceUpdate, HyperparameterSearchSpaceUpdates, parse_hyperparameter_search_space_updates ) @@ -317,13 +318,20 @@ def test_pipeline_score(fit_dictionary_tabular_dummy): given the default configuration""" # increase number of epochs to test for performance fit_dictionary_tabular_dummy['epochs'] = 50 - fit_dictionary_tabular_dummy['early_stopping'] = 30 + fit_dictionary_tabular_dummy['early_stopping'] = -1 X = fit_dictionary_tabular_dummy['X_train'].copy() y = fit_dictionary_tabular_dummy['y_train'].copy() pipeline = TabularRegressionPipeline( dataset_properties=fit_dictionary_tabular_dummy['dataset_properties'], + search_space_updates=HyperparameterSearchSpaceUpdates([ + HyperparameterSearchSpaceUpdate("optimizer", + "AdamOptimizer:lr", + value_range=[0.0001, 0.001], + default_value=0.001)] + ), + exclude={'trainer': ['AdversarialTrainer']}, random_state=2 ) @@ -339,5 +347,5 @@ def test_pipeline_score(fit_dictionary_tabular_dummy): r2_score = pipeline.score(X, y) # we should be able to get a decent score on this dummy data - assert r2_score >= 0.8, f"Pipeline:{pipeline} Config:{config} FitDict: {fit_dictionary_tabular_dummy}, " \ + assert r2_score >= 0.5, f"Pipeline:{pipeline} Config:{config} FitDict: {fit_dictionary_tabular_dummy}, " \ f"{pipeline.named_steps['trainer'].run_summary.performance_tracker['train_metrics']}"