diff --git a/autorecsys/auto_search.py b/autorecsys/auto_search.py deleted file mode 100644 index 4c0b7ad..0000000 --- a/autorecsys/auto_search.py +++ /dev/null @@ -1,142 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -import logging -import tempfile -import tensorflow as tf - -from autorecsys.utils.common import to_snake_case, create_directory, load_dataframe_input -from autorecsys.searcher.tuners.tuner import METRIC, PipeTuner -from autorecsys.searcher import tuners -from autorecsys.recommender import CTRRecommender, RPRecommender - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') -logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -class Search(object): - """ A search object to search on a Recommender HyperModel (CTRRecommender/RPRecommender) - defined by inputs and outputs. - - ``Search`` combines a Recommender and a Tuner to tune the Recommender. The user can - use ``search()`` to perform search, and use a similar way to a Keras model to adopt - the best discovered model as it also has `fit()`/`predict()`/`evaluate()` methods. - The user should input a Recommender HyperModel (CTRRecommender/RPRecommender) and a - selected tuning method to initial the ``Search`` object and input the dataset when - calling the ``search`` method to discover the best architecture. - ``` - # Arguments - model: A Recommender HyperModel (CTRRecommender/RPRecommender). - name: String. The name of the project, which is used for saving and loading purposes. - tuner: String. The name of the tuner. It should be one of 'greedy', 'bayesian' or - 'random'. Default to be 'random'. - - - tuner_params: Dict. The hyperparameters of the tuner. The commons ones are: - 'max_trials': Int. Specify the number of search epochs. - 'overwrite': Boolean. Whether we want to ovewrite an existing - tuner or not. - - directory: String. The path to a directory for storing the search outputs. - Defaults to None, which would create a folder with the name of the - project in the current directory, i.e., ``directory/name``. - overwrite: Boolean. Defaults to `True`. Whether we want to ovewrite an existing - project with the name defined as ``directory/name`` or not. - """ - def __init__(self, model=None, name=None, tuner='random', tuner_params=None, directory='.', overwrite=True): - self.pipe = model - self.tuner = tuner - self.tuner_params = tuner_params - if not name: - prefix = self.__class__.__name__ - name = prefix + '_' + str(tf.keras.backend.get_uid(prefix)) - name = to_snake_case(name) - self.name = name - directory = directory or tempfile.gettempdir() - self.dir = os.path.join(directory, self.name) - - self.overwrite = overwrite - create_directory(self.dir, remove_existing=overwrite) - self.logger = logging.getLogger(self.name) - self.logger.info('Project directory: {}'.format(self.dir)) - self.best_keras_graph = None - self.best_model = None - self.need_fully_train = False - - def search(self, x=None, y=None, x_val=None, y_val=None, objective='mse', **fit_kwargs): - """Search the best deep recommendation model. - - # Arguments - x: numpy array. Training features. - y: numpy array. Training targets. - x_val: numpy array. Validation features. - y_val: numpy array. Validation features. - objective: String. Name of model metric to minimize or maximize, - e.g. 'val_BinaryCrossentropy'. Defaults to 'mse'. - **fit_kwargs: Any arguments supported by the fit method of a Keras model such as: - ``batch_size``, ``epochs``, ``callbacks``. - """ - - # overwrite the objective - self.objective = objective - tuner = self._build_tuner(self.tuner, self.tuner_params) - - # TODO search on a small piece of train data, currently it uses whole train data - tuner.search(x=x, y=y, x_val=x_val, y_val=y_val, **fit_kwargs) - # show the search space - tuner.search_space_summary() - # show the search results - tuner.results_summary() - best_pipe_lists = tuner.get_best_models(1) - # len(best_pipe_lists) == 0 means that this pipeline does not have tunable parameters - self.best_model = best_pipe_lists[0] - return self.best_model - - def _build_tuner(self, tuner, tuner_params): - """Build a tuner based on its name and hyperparameters. - - # Arguments - tuner: String. The name of the tuner. It should be one of 'greedy', 'bayesian' or - 'random'. Default to be 'random'. - - tuner_params: Dict. The hyperparameters of the tuner. The commons ones are: - 'max_trials': Int. Specify the number of search epochs. - 'overwrite': Boolean. Whether we want to ovewrite an existing - tuner or not. - """ - tuner_cls = tuners.get_tuner_class( tuner ) - hps = self.pipe.get_hyperparameters() - tuner = tuner_cls(hypergraph=self.pipe, - objective=self.objective, - hyperparameters=hps, - directory=self.dir, - **tuner_params) - return tuner - - def predict(self, x): - """Use the best searched model to conduct prediction on the dataset x. - - # Arguments - x: numpy array / data frame / string path of a csv file. - Features used to do the prediction. - """ - if isinstance (self.pipe, RPRecommender): - x = load_dataframe_input(x) - return self.best_model.predict(x) - - def evaluate(self, x, y_true): - """Evaluate the best searched model. - - # Arguments - x: numpy array / data frame / string path of a csv file. - Features used to do the prediction. - y_true: numpy array / data frame / string path of a csv file. - Ground-truth labels. - """ - y_pred = self.predict(x) - score_func = METRIC[self.objective.split('_')[-1]] - y_true = load_dataframe_input(y_true) - y_true = y_true.values.reshape(-1, 1) - self.logger.info(f'evaluate prediction results using {self.objective}') - return score_func(y_true, y_pred) diff --git a/autorecsys/pipeline/__init__.py b/autorecsys/pipeline/__init__.py index e2f7fee..eb8937b 100644 --- a/autorecsys/pipeline/__init__.py +++ b/autorecsys/pipeline/__init__.py @@ -8,4 +8,4 @@ HyperInteraction from autorecsys.pipeline.optimizer import RatingPredictionOptimizer, PointWiseOptimizer -from autorecsys.pipeline.node import Input, StructuredDataInput +#from autorecsys.pipeline.node import Input, StructuredDataInput diff --git a/autorecsys/pipeline/base.py b/autorecsys/pipeline/base.py deleted file mode 100644 index 6e78c51..0000000 --- a/autorecsys/pipeline/base.py +++ /dev/null @@ -1,266 +0,0 @@ -import types -import tensorflow as tf -from autorecsys.searcher.core import hyperparameters as hp_module -from autorecsys.searcher.core.trial import Stateful -from autorecsys.utils.common import to_snake_case -from tensorflow.python.util import nest - - -class Node(Stateful): - """The nodes in a network connecting the blocks.""" - - def __init__(self, shape=None): - super().__init__() - self.in_blocks = [] - self.out_blocks = [] - self.shape = shape - - def add_in_block(self, hypermodel): - self.in_blocks.append(hypermodel) - - def add_out_block(self, hypermodel): - self.out_blocks.append(hypermodel) - - def build(self): - return tf.keras.Input(shape=self.shape) - - def get_state(self): - return {'shape': self.shape} - - def set_state(self, state): - self.shape = state['shape'] - - -class HyperModel(object): - """Defines a searchable space of Models and builds Models from this space. - # Attributes: - name: The name of this HyperModel. - tunable: Whether the hyperparameters defined in this hypermodel - should be added to search space. If `False`, either the search - space for these parameters must be defined in advance, or the - default values will be used. - """ - - def __init__(self, name=None, tunable=True): - self.name = name - self.tunable = tunable - - self._build = self.build - self.build = self._build_wrapper - - def build(self, hp): - """Builds a model. - # Arguments: - hp: A `HyperParameters` instance. - # Returns: - A model instance. - """ - raise NotImplementedError - - def _build_wrapper(self, hp, *args, **kwargs): - if not self.tunable: - # Copy `HyperParameters` object so that new entries are not added - # to the search space. - hp = hp.copy() - return self._build(hp, *args, **kwargs) - - -class Block(HyperModel, Stateful): - def __init__(self, name=None, **kwargs): - super().__init__(**kwargs) - self.fixed_params = None - self.tunable_candidates = None - if not name: - prefix = self.__class__.__name__ - name = prefix + '_' + str(tf.keras.backend.get_uid(prefix)) - name = to_snake_case(name) - self._hyperparameters = None - self.name = name - self.inputs = None - self.outputs = None - self._num_output_node = 1 - - def __new__(cls, *args, **kwargs): - obj = super().__new__(cls) - build_fn = obj.build - - def build_wrapper(obj, hp, *args, **kwargs): - with hp.name_scope(obj.name): - return build_fn(hp, *args, **kwargs) - - obj.build = types.MethodType(build_wrapper, obj) - return obj - - def __str__(self): - return self.name - - @property - def hyperparameters(self): - return self._hyperparameters - - def __call__(self, inputs): - """Functional API. - # Arguments - inputs: A list of input node(s) or a single input node for the block. - # Returns - list: A list of output node(s) of the Block. - """ - inputs = nest.flatten(inputs) - self.inputs = inputs - for input_node in self.inputs: - if not isinstance(input_node, Node): - raise TypeError('Expect the inputs to layer {name} to be ' - 'a Node, but got {type}.'.format( - name=self.name, - type=type(input_node))) - input_node.add_out_block(self) - self.outputs = [] - for _ in range(self._num_output_node): - output_node = Node() - output_node.add_in_block(self) - self.outputs.append(output_node) - return self.outputs - - def get_state(self): - """Get the configuration of the preprocessor. - # Returns - A dictionary of configurations of the preprocessor. - """ - return {'name': self.name} - - def set_state(self, state): - """Set the configuration of the preprocessor. - # Arguments - state: A dictionary of the configurations of the preprocessor. - """ - if 'name' in state: - self.name = state['name'] - - -class HyperBlock(Block): - """HyperBlock uses hyperparameters to decide inner Block graph. - A HyperBlock should be build into connected Blocks instead of individual Keras - layers. The main purpose of creating the HyperBlock class is for the ease of - parsing the graph for preprocessors. The graph would be hard to parse if a Block, - whose inner structure is decided by hyperparameters dynamically, contains both - preprocessors and Keras layers. - When the preprocessing layers of Keras are ready to cover all the preprocessors - in AutoKeras, the preprocessors should be handled by the Keras Model. The - HyperBlock class should be removed. The subclasses should extend Block class - directly and the build function should build connected Keras layers instead of - Blocks. - # Arguments - output_shape: Tuple of int(s). Defaults to None. If None, the output shape - will be inferred from the AutoModel. - name: String. The name of the block. If unspecified, it will be set - automatically with the class name. - """ - - def __init__(self, output_shape=None, **kwargs): - super().__init__(**kwargs) - self.output_shape = output_shape - - def build(self, hp, inputs=None): - """Build the HyperModel instead of Keras Model. - # Arguments - hp: HyperParameters. The hyperparameters for building the model. - inputs: A list of instances of Node. - # Returns - An Node instance, the output node of the output Block. - """ - raise NotImplementedError - - -class Preprocessor(Block): - """Hyper preprocessing block base class. - It extends Block which extends Hypermodel. A preprocessor is a Hypermodel, which - means it is a search space. However, different from other Hypermodels, it is - also a model which can be fit. - """ - - def build(self, hp): - """Get the values of the required HyperParameters. - It does not build and return a Keras Model, but initialize the - HyperParameters for the preprocessor to be fit. - """ - pass - - def update(self, x, y=None): - """Incrementally fit the preprocessor with a single training instance. - # Arguments - x: EagerTensor. A single instance in the training dataset. - y: EagerTensor. The targets of the tasks. Defaults to None. - """ - raise NotImplementedError - - def transform(self, x, fit=False): - """Incrementally fit the preprocessor with a single training instance. - # Arguments - x: EagerTensor. A single instance in the training dataset. - fit: Boolean. Whether it is in fit mode. - Returns: - A transformed instanced which can be converted to a tf.Tensor. - """ - raise NotImplementedError - - def output_types(self): - """The output types of the transformed data, e.g. tf.int64. - The output types are required by tf.py_function, which is used for transform - the dataset into a new one with a map function. - # Returns - A tuple of data types. - """ - raise NotImplementedError - - @property - def output_shape(self): - """The output shape of the transformed data. - The output shape is needed to build the Keras Model from the AutoModel. - The output shape of the preprocessor is the input shape of the Keras Model. - # Returns - A tuple of int(s) or a TensorShape. - """ - raise NotImplementedError - - def finalize(self): - """Training process of the preprocessor after update with all instances.""" - pass - - def get_config(self): - """Get the configuration of the preprocessor. - # Returns - A dictionary of configurations of the preprocessor. - """ - return {} - - def set_config(self, config): - """Set the configuration of the preprocessor. - # Arguments - config: A dictionary of the configurations of the preprocessor. - """ - pass - - def get_weights(self): - """Get the trained weights of the preprocessor. - # Returns - A dictionary of trained weights of the preprocessor. - """ - return {} - - def set_weights(self, weights): - """Set the trained weights of the preprocessor. - # Arguments - weights: A dictionary of trained weights of the preprocessor. - """ - pass - - def get_state(self): - state = super().get_state() - state.update(self.get_config()) - return {'config': state, - 'weights': self.get_weights()} - - def set_state(self, state): - self.set_config(state['config']) - super().set_state(state['config']) - self.set_weights(state['weights']) diff --git a/autorecsys/pipeline/graph.py b/autorecsys/pipeline/graph.py deleted file mode 100644 index c7dd5ef..0000000 --- a/autorecsys/pipeline/graph.py +++ /dev/null @@ -1,470 +0,0 @@ -import os -import pickle -import functools - -from autorecsys.searcher.core.trial import Stateful -from autorecsys.searcher.core import hyperparameters as hp_module -from autorecsys.pipeline import base - -import tensorflow as tf -from tensorflow.python.util import nest - - -class Graph(Stateful): - """A graph consists of connected Blocks, HyperBlocks - - # Arguments - inputs: A list of input node(s) for the Graph. - outputs: A list of output node(s) for the Graph. - """ - def __init__(self, inputs, outputs): - super().__init__() - # TODO flatten inputs & outputs - self.inputs = nest.flatten(inputs) - self.outputs = nest.flatten(outputs) - # reverse order of the topological sort - self._node_to_id = {} - self._nodes = [] - # topological sort of the blocks in the graph - self._blocks = [] - self._block_to_id = {} - self._build_network() - - def compile(self, func): - """Share the information between blocks by calling functions in compiler. - - # Arguments - func: A dictionary. The keys are the block classes. The values are - corresponding compile functions. - """ - for block in self._blocks: - if block.__class__ in func: - func[block.__class__](block) - - - def _build_network(self): - self._node_to_id = {} - - # Recursively find all the interested nodes. - for input_node in self.inputs: - self._search_network(input_node, self.outputs, set(), set()) - # the topological sort of the graph in reverse order - self._nodes = sorted(list(self._node_to_id.keys()), - key=lambda x: self._node_to_id[x]) - - for node in (self.inputs + self.outputs): - if node not in self._node_to_id: - raise ValueError('Inputs and outputs not connected.') - - # Find the blocks. - blocks = [] - for input_node in self._nodes: - for block in input_node.out_blocks: - if any([output_node in self._node_to_id - for output_node in block.outputs]) and block not in blocks: - blocks.append(block) - - # Check if all the inputs of the blocks are set as inputs. - for block in blocks: - for input_node in block.inputs: - if input_node not in self._node_to_id: - raise ValueError('A required input is missing for HyperModel ' - '{name}.'.format(name=block.name)) - - # Calculate the in degree of all the nodes - in_degree = [0] * len(self._nodes) - for node_id, node in enumerate(self._nodes): - in_degree[node_id] = len([ - block for block in node.in_blocks if block in blocks]) - - # Add the blocks in topological order. - self._blocks = [] - self._block_to_id = {} - while len(blocks) != 0: - new_added = [] - - # Collect blocks with in degree 0. - for block in blocks: - if any([in_degree[self._node_to_id[node]] - for node in block.inputs]): - continue - new_added.append(block) - - # Remove the collected blocks from blocks. - for block in new_added: - blocks.remove(block) - - for block in new_added: - # Add the collected blocks to the AutoModel. - self._add_block(block) - - # Decrease the in degree of the output nodes. - for output_node in block.outputs: - if output_node not in self._node_to_id: - continue - output_node_id = self._node_to_id[output_node] - in_degree[output_node_id] -= 1 - - def _search_network(self, input_node, outputs, in_stack_nodes, - visited_nodes): - visited_nodes.add(input_node) - in_stack_nodes.add(input_node) - - outputs_reached = False - if input_node in outputs: - outputs_reached = True - - for block in input_node.out_blocks: - for output_node in block.outputs: - if output_node in in_stack_nodes: - raise ValueError('The network has a cycle.') - if output_node not in visited_nodes: - self._search_network(output_node, outputs, in_stack_nodes, - visited_nodes) - if output_node in self._node_to_id.keys(): - outputs_reached = True - - if outputs_reached: - self._add_node(input_node) - - in_stack_nodes.remove(input_node) - - def _add_block(self, block): - if block not in self._blocks: - block_id = len(self._blocks) - self._block_to_id[block] = block_id - self._blocks.append(block) - - def _add_node(self, input_node): - if input_node not in self._node_to_id: - self._node_to_id[input_node] = len(self._node_to_id) - - def _get_block(self, name): - for block in self._blocks: - if block.name == name: - return block - raise ValueError('Cannot find block named {name}.'.format(name=name)) - - def get_state(self): - block_state = {str(block_id): block.get_state() - for block_id, block in enumerate(self._blocks)} - node_state = {str(node_id): node.get_state() - for node_id, node in enumerate(self._nodes)} - return {'blocks': block_state, 'nodes': node_state} - - def set_state(self, state): - block_state = state['blocks'] - node_state = state['nodes'] - for block_id, block in enumerate(self._blocks): - block.set_state(block_state[str(block_id)]) - for node_id, node in enumerate(self._nodes): - node.set_state(node_state[str(node_id)]) - - def save(self, fname): - state = self.get_state() - with tf.io.gfile.GFile(fname, 'wb') as f: - pickle.dump(state, f) - return str(fname) - - def reload(self, fname): - with tf.io.gfile.GFile(fname, 'rb') as f: - state = pickle.load(f) - self.set_state(state) - - def build(self, hp): - pass - -class PreprocessGraph(Graph): - """A graph consists of only Preprocessors. - It is both a search space with Hyperparameters and a model to be fitted. It - preprocess the dataset with the Preprocessors. The output is the input to the - Keras model. It does not extend Hypermodel class because it cannot be built into - a Keras model. - """ - - def preprocess(self, dataset, validation_data=None, fit=False): - """Preprocess the data to be ready for the Keras Model. - # Arguments - dataset: tf.data.Dataset. Training data. - validation_data: tf.data.Dataset. Validation data. - fit: Boolean. Whether to fit the preprocessing layers with x and y. - # Returns - if validation data is provided. - A tuple of two preprocessed tf.data.Dataset, (train, validation). - Otherwise, return the training dataset. - """ - dataset = self._preprocess(dataset, fit=fit) - if validation_data: - validation_data = self._preprocess(validation_data) - return dataset, validation_data - - def _preprocess(self, dataset, fit=False): - # A list of input node ids in the same order as the x in the dataset. - input_node_ids = [self._node_to_id[input_node] for input_node in self.inputs] - - # Iterate until all the model inputs have their data. - while set(map(lambda node: self._node_to_id[node], self.outputs) - ) - set(input_node_ids): - # Gather the blocks for the next iteration over the dataset. - blocks = [] - for node_id in input_node_ids: - for block in self._nodes[node_id].out_blocks: - if block in self._blocks: - blocks.append(block) - if fit: - # Iterate the dataset to fit the preprocessors in current depth. - self._fit(dataset, input_node_ids, blocks) - - # Transform the dataset. - output_node_ids = [] - dataset = dataset.map(functools.partial( - self._transform, - input_node_ids=input_node_ids, - output_node_ids=output_node_ids, - blocks=blocks, - fit=fit)) - - # Build input_node_ids for next depth. - input_node_ids = output_node_ids - return dataset - - def _fit(self, dataset, input_node_ids, blocks): - # Iterate the dataset to fit the preprocessors in current depth. - for x, y in dataset: - x = nest.flatten(x) - id_to_data = { - node_id: temp_x for temp_x, node_id in zip(x, input_node_ids) - } - for block in blocks: - data = [id_to_data[self._node_to_id[input_node]] - for input_node in block.inputs] - block.update(data, y=y) - - # Finalize and set the shapes of the output nodes. - for block in blocks: - block.finalize() - nest.flatten(block.outputs)[0].shape = block.output_shape - - def _transform(self, - x, - y, - input_node_ids, - output_node_ids, - blocks, - fit=False): - x = nest.flatten(x) - id_to_data = { - node_id: temp_x - for temp_x, node_id in zip(x, input_node_ids) - } - output_data = {} - # Transform each x by the corresponding block. - for hm in blocks: - data = [id_to_data[self._node_to_id[input_node]] - for input_node in hm.inputs] - data = tf.py_function(functools.partial(hm.transform, fit=fit), - inp=nest.flatten(data), - Tout=hm.output_types()) - data = nest.flatten(data)[0] - data.set_shape(hm.output_shape) - output_data[self._node_to_id[hm.outputs[0]]] = data - # Keep the Keras Model inputs even they are not inputs to the blocks. - for node_id, data in id_to_data.items(): - if self._nodes[node_id] in self.outputs: - output_data[node_id] = data - - for node_id in sorted(output_data.keys()): - output_node_ids.append(node_id) - return tuple(map( - lambda node_id: output_data[node_id], output_node_ids)), y - - def build(self, hp): - """Obtain the values of all the HyperParameters. - Different from the build function of Hypermodel. This build function does not - produce a Keras model. It only obtain the hyperparameter values from - HyperParameters. - # Arguments - hp: HyperParameters. - """ - super().build(hp) - # self.compile(compiler.BEFORE) - for block in self._blocks: - block.build(hp) - - -class KerasGraph(Graph, base.HyperModel): - """A graph and HyperModel to be built into a Keras model.""" - - def build(self, hp): - """Build the HyperModel into a Keras Model.""" - super().build(hp) - # self.compile(compiler.AFTER) - real_nodes = {} - for input_node in self.inputs: - node_id = self._node_to_id[input_node] - real_nodes[node_id] = input_node.build() - for block in self._blocks: - temp_inputs = [real_nodes[self._node_to_id[input_node]] - for input_node in block.inputs] - outputs = block.build(hp, inputs=temp_inputs) - outputs = nest.flatten(outputs) - for output_node, real_output_node in zip(block.outputs, outputs): - real_nodes[self._node_to_id[output_node]] = real_output_node - model = tf.keras.Model( - [real_nodes[self._node_to_id[input_node]] for input_node in - self.inputs], - [real_nodes[self._node_to_id[output_node]] for output_node in - self.outputs]) - - return self._compile_keras_model(hp, model) - - def _get_metrics(self): - # metrics = {} - metrics = [] - for output_node in self.outputs: - block = output_node.in_blocks[0] - if 'optimizer' in str(type(block)): - # metrics[block.name] = block.metric - metrics.append(block.metric) - return metrics - - def _get_loss(self): - # loss = {} - loss = [] - for output_node in self.outputs: - block = output_node.in_blocks[0] - if 'optimizer' in str(type(block)): - # loss[block.name] = block.loss - loss.append(block.loss) - return loss - - def _compile_keras_model(self, hp, model): - # Specify hyperparameters from compile(...) - optimizer = hp.Choice('optimizer', - ['adam', - # 'adadelta', - # "Adagrad", - # "RMSprop", - # "AdaMax", - # 'sgd' - ]) - - model.compile(optimizer=optimizer, - metrics=self._get_metrics(), - loss=self._get_loss()) - - return model - - -class PlainGraph(Graph): - """A graph built from a HyperGraph to produce KerasGraph and PreprocessGraph. - A PlainGraph does not contain HyperBlock. HyperGraph's hyper_build function - returns an instance of PlainGraph, which can be directly built into a KerasGraph - and a PreprocessGraph. - # Arguments - inputs: A list of input node(s) for the PlainGraph. - outputs: A list of output node(s) for the PlainGraph. - """ - - def __init__(self, inputs, outputs, **kwargs): - self._keras_model_inputs = [] - super().__init__(inputs=inputs, outputs=outputs, **kwargs) - - def _build_network(self): - super()._build_network() - # Find the model input nodes - for node in self._nodes: - if self._is_keras_model_inputs(node): - self._keras_model_inputs.append(node) - - self._keras_model_inputs = sorted(self._keras_model_inputs, - key=lambda x: self._node_to_id[x]) - - @staticmethod - def _is_keras_model_inputs(node): - for block in node.in_blocks: - if not isinstance(block, base.Preprocessor): - return False - for block in node.out_blocks: - if not isinstance(block, base.Preprocessor): - return True - return False - - def build_keras_graph(self): - return KerasGraph(self._keras_model_inputs, - self.outputs) - - def build_preprocess_graph(self): - return PreprocessGraph(self.inputs, - self._keras_model_inputs) - - -def copy(old_instance): - instance = old_instance.__class__() - instance.set_state(old_instance.get_state()) - return instance - - - -class HyperGraph(Graph): - """A HyperModel based on connected Blocks and HyperBlocks. - # Arguments - inputs: A list of input node(s) for the HyperGraph. - outputs: A list of output node(s) for the HyperGraph. - """ - - def __init__(self, inputs, outputs, **kwargs): - super().__init__(inputs, outputs, **kwargs) - # self.compile(compiler.HYPER) - - def build_graphs(self, hp): - plain_graph = self.hyper_build(hp) - return plain_graph.build_keras_graph() - - def save_weights(self, directory): - for block in self._blocks: - block_filename = os.path.join(directory, block.name) - block.save_weights(filename=block_filename) - - def get_hyperparameters(self): - """Get the tunable hyperperparameters from all the blocks in this pipeline.""" - hps = hp_module.HyperParameters() - for block in self._blocks: - params_dict = block.hyperparameters - if params_dict: - with hps.name_scope(block.name): - for param_name, single_hp in params_dict.items(): - hps.register(param_name, - single_hp.__class__.__name__, - single_hp.get_config()) - return hps - - - def hyper_build(self, hp): - """Build a GraphHyperModel with no HyperBlock but only Block.""" - # Make sure get_uid would count from start. - tf.keras.backend.clear_session() - inputs = [] - old_node_to_new = {} - for old_input_node in self.inputs: - input_node = copy(old_input_node) - inputs.append(input_node) - old_node_to_new[old_input_node] = input_node - for old_block in self._blocks: - inputs = [old_node_to_new[input_node] - for input_node in old_block.inputs] - if isinstance(old_block, base.HyperBlock): - outputs = old_block.build(hp, inputs=inputs) - else: - outputs = copy(old_block)(inputs) - for output_node, old_output_node in zip(outputs, old_block.outputs): - old_node_to_new[old_output_node] = output_node - inputs = [] - for input_node in self.inputs: - inputs.append(old_node_to_new[input_node]) - outputs = [] - for output_node in self.outputs: - outputs.append(old_node_to_new[output_node]) - - pipe = PlainGraph(inputs, outputs) - return pipe diff --git a/autorecsys/pipeline/interactor.py b/autorecsys/pipeline/interactor.py index fb582d6..2ca7f5c 100644 --- a/autorecsys/pipeline/interactor.py +++ b/autorecsys/pipeline/interactor.py @@ -2,7 +2,8 @@ import tensorflow as tf from tensorflow.python.util import nest -from autorecsys.pipeline.base import Block +#from autorecsys.pipeline.base import Block +from autokeras.engine.block import Block from autorecsys.pipeline.utils import Bias import random import tensorflow as tf diff --git a/autorecsys/pipeline/mapper.py b/autorecsys/pipeline/mapper.py index 774c2c8..8a63517 100644 --- a/autorecsys/pipeline/mapper.py +++ b/autorecsys/pipeline/mapper.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, division, print_function, unicode_literals import tensorflow as tf -from autorecsys.pipeline.base import Block +from autokeras.engine.block import Block +#from autorecsys.pipeline.base import Block class LatentFactorMapper(Block): diff --git a/autorecsys/pipeline/node.py b/autorecsys/pipeline/node.py deleted file mode 100644 index 4c4018e..0000000 --- a/autorecsys/pipeline/node.py +++ /dev/null @@ -1,184 +0,0 @@ -import numpy as np -import pandas as pd -import tensorflow as tf -from tensorflow.python.util import nest - -from autorecsys.utils.common import dataset_shape -from autorecsys.pipeline import base - - - -class Input(base.Node): - """Input node for tensor data. - The data should be numpy.ndarray or tf.data.Dataset. - """ - - def _check(self, x): - """Record any information needed by transform.""" - if not isinstance(x, (np.ndarray, tf.data.Dataset)): - raise TypeError('Expect the data to Input to be numpy.ndarray or ' - 'tf.data.Dataset, but got {type}.'.format(type=type(x))) - if isinstance(x, np.ndarray) and not np.issubdtype(x.dtype, np.number): - raise TypeError('Expect the data to Input to be numerical, but got ' - '{type}.'.format(type=x.dtype)) - - def _convert_to_dataset(self, x): - if isinstance(x, tf.data.Dataset): - return x - if isinstance(x, np.ndarray): - x = x.astype(np.float32) - return tf.data.Dataset.from_tensor_slices(x) - - def _record_dataset_shape(self, dataset): - self.shape = dataset_shape(dataset) - - def fit_transform(self, x): - dataset = self.transform(x) - self._record_dataset_shape(dataset) - return dataset - - def transform(self, x): - """Transform x into a compatible type (tf.data.Dataset).""" - self._check(x) - dataset = self._convert_to_dataset(x) - return dataset - - -class StructuredDataInput(Input): - """Input node for structured data. - The input data should be numpy.ndarray, pandas.DataFrame or tensorflow.Dataset. - # Arguments - column_names: A list of strings specifying the names of the columns. The - length of the list should be equal to the number of columns of the data. - Defaults to None. If None, it will obtained from the header of the csv - file or the pandas.DataFrame. - column_types: Dict. The keys are the column names. The values should either - be 'numerical' or 'categorical', indicating the type of that column. - Defaults to None. If not None, the column_names need to be specified. - If None, it will be inferred from the data. A column will be judged as - categorical if the number of different values is less than 5% of the - number of instances. - """ - - def __init__(self, column_names=None, column_types=None, **kwargs): - super().__init__(**kwargs) - self.column_names = column_names - self.column_types = column_types - # Variables for inferring column types. - self.count_nan = None - self.count_numerical = None - self.count_categorical = None - self.count_unique_numerical = [] - self.num_col = None - - def get_state(self): - state = super().get_state() - state.update({ - 'column_names': self.column_names, - 'column_types': self.column_types, - 'count_nan': self.count_nan, - 'count_numerical': self.count_numerical, - 'count_categorical': self.count_categorical, - 'count_unique_numerical': self.count_unique_numerical, - 'num_col': self.num_col - }) - return state - - def set_state(self, state): - super().set_state(state) - self.column_names = state['column_names'] - self.column_types = state['column_types'] - self.count_nan = state['count_nan'] - self.count_numerical = state['count_numerical'] - self.count_categorical = state['count_categorical'] - self.count_unique_numerical = state['count_unique_numerical'] - self.num_col = state['num_col'] - - def _check(self, x): - if not isinstance(x, (pd.DataFrame, np.ndarray)): - raise TypeError('Unsupported type {type} for ' - '{name}.'.format(type=type(x), - name=self.__class__.__name__)) - - # Extract column_names from pd.DataFrame. - if isinstance(x, pd.DataFrame) and self.column_names is None: - self.column_names = list(x.columns) - # column_types is provided by user - if self.column_types: - for column_name in self.column_types: - if column_name not in self.column_names: - raise ValueError('Column_names and column_types are ' - 'mismatched. Cannot find column name ' - '{name} in the data.'.format( - name=column_name)) - - # Generate column_names. - if self.column_names is None: - if self.column_types: - raise ValueError('Column names must be specified.') - self.column_names = [index for index in range(x.shape[1])] - - # Check if column_names has the correct length. - if len(self.column_names) != x.shape[1]: - raise ValueError('Expect column_names to have length {expect} ' - 'but got {actual}.'.format( - expect=x.shape[1], - actual=len(self.column_names))) - - def _convert_to_dataset(self, x): - if isinstance(x, pd.DataFrame): - # Convert x, y, validation_data to tf.Dataset. - x = tf.data.Dataset.from_tensor_slices( - x.values.astype(np.unicode)) - if isinstance(x, np.ndarray): - x = tf.data.Dataset.from_tensor_slices(x.astype(np.unicode)) - dataset = super()._convert_to_dataset(x) - for x in dataset: - self.update(x) - self.infer_column_types() - return dataset - - def update(self, x): - # Calculate the statistics. - x = nest.flatten(x)[0].numpy() - if self.num_col is None: - self.num_col = len(x) - self.count_nan = np.zeros(self.num_col) - self.count_numerical = np.zeros(self.num_col) - self.count_categorical = np.zeros(self.num_col) - for i in range(len(x)): - self.count_unique_numerical.append({}) - for i in range(self.num_col): - x[i] = x[i].decode('utf-8') - if x[i] == 'nan': - self.count_nan[i] += 1 - elif x[i] == 'True': - self.count_categorical[i] += 1 - elif x[i] == 'False': - self.count_categorical[i] += 1 - else: - try: - tmp_num = float(x[i]) - self.count_numerical[i] += 1 - if tmp_num not in self.count_unique_numerical[i]: - self.count_unique_numerical[i][tmp_num] = 1 - else: - self.count_unique_numerical[i][tmp_num] += 1 - except ValueError: - self.count_categorical[i] += 1 - - def infer_column_types(self): - column_types = {} - for i in range(self.num_col): - if self.count_categorical[i] > 0: - column_types[self.column_names[i]] = 'categorical' - elif len(self.count_unique_numerical[i])/self.count_numerical[i] < 0.05: - column_types[self.column_names[i]] = 'categorical' - else: - column_types[self.column_names[i]] = 'numerical' - # Partial column_types is provided. - if self.column_types is None: - self.column_types = {} - for key, value in column_types.items(): - if key not in self.column_types: - self.column_types[key] = value diff --git a/autorecsys/pipeline/optimizer.py b/autorecsys/pipeline/optimizer.py index 3929b3f..f4c8978 100644 --- a/autorecsys/pipeline/optimizer.py +++ b/autorecsys/pipeline/optimizer.py @@ -1,7 +1,8 @@ from __future__ import absolute_import, division, print_function, unicode_literals import tensorflow as tf -from autorecsys.pipeline.base import Block +#from autorecsys.pipeline.base import Block +from autokeras.engine.block import Block class RatingPredictionOptimizer(Block): diff --git a/autorecsys/pipeline/preprocessor.py b/autorecsys/pipeline/preprocessor.py index 6ab2fa9..0037b6e 100644 --- a/autorecsys/pipeline/preprocessor.py +++ b/autorecsys/pipeline/preprocessor.py @@ -17,6 +17,7 @@ import math from pathlib import Path from autorecsys.utils.common import load_pickle, save_pickle +from datetime import datetime class BaseProprocessor(metaclass=ABCMeta): @@ -137,7 +138,7 @@ def __init__(self, dataset_path="./examples/datasets/avazu/sampled_train_10000.t # which of these info's corresponding instance variables are and pre-set them with constant values. self.column_names = ['id', 'click', 'hour', 'C1', 'banner_pos', 'site_id', 'site_domain', 'site_category', 'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip', 'device_model', - 'device_type', 'device_conn_type', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21'] + 'device_type', 'device_conn_type', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21'] self.used_column_names = None self.pd_data = None self.hash_sizes = None @@ -149,7 +150,6 @@ def __init__(self, dataset_path="./examples/datasets/avazu/sampled_train_10000.t # self.used_columns_names = self.columns_names # self.dtype_dict = {n: np.float32 for n in self.used_columns_names} - def load_data(self): """ Set the following variables: @@ -436,7 +436,7 @@ def scale_numerical_data(self): def scale_by_natural_log(num): # TODO: 1) Explain why the conditional statement makes exception for numbers like 1, where 1>ln(1)**2 if num > 2: - num = math.log(float(num))**2 + num = math.log(float(num)) ** 2 return num for numer_name in self.numer_names: @@ -489,6 +489,7 @@ def preprocessing(self, train_size, validate_size, random_state=42): return train_X, train_y, validate_X, validate_y, test_X, test_y + class NetflixPrizePreprocessor(BaseRatingPredictionProprocessor): def __init__(self, dataset_path): @@ -512,7 +513,7 @@ def _load_data(self): with open(fp, 'r') as f: for line in f.readlines(): if ':' in line: - movie = int(line.strip(":\n"))-1 # -1 because the sequential MovieIDs starts from 1 + movie = int(line.strip(":\n")) - 1 # -1 because the sequential MovieIDs starts from 1 else: user, rating = [int(v) for v in line.strip().split(',')[:2]] cols[1].append(movie) @@ -537,13 +538,75 @@ def _load_data(self): def preprocessing(self, val_test_size, random_state): self.X = self.pd_data.iloc[::, :-1].values self.y = self.pd_data.iloc[::, [-1]].values - self.train_X, self.val_test_X, self.train_y, self.val_test_y = train_test_split(self.X, self.y, test_size = val_test_size * 2, - random_state=random_state) - self.val_X, self.test_X, self.val_y, self.test_y = train_test_split(self.val_test_X, self.val_test_y, test_size = 0.5, - random_state=random_state) + self.train_X, self.val_test_X, self.train_y, self.val_test_y = train_test_split(self.X, self.y, + test_size=val_test_size * 2, + random_state=random_state) + self.val_X, self.test_X, self.val_y, self.test_y = train_test_split(self.val_test_X, self.val_test_y, + test_size=0.5, + random_state=random_state) + +# here class MovielensPreprocessor(BaseRatingPredictionProprocessor): + used_columns_names: List[str] + + def __init__(self, dataset_path, time=False, sep='::'): + super(MovielensPreprocessor, self).__init__(dataset_path=dataset_path, ) + self.time = time + self.columns_names = ["user_id", "item_id", "rating", "timestamp"] + self.used_columns_names = ["user_id", "item_id", "rating"] + self.dtype_dict = {"user_id": np.float32, "item_id": np.float32, "rating": np.float32, "timestamp": np.float32} + self.sep = sep + self._load_data() + def _load_data(self): + self.pd_data = pd.read_csv(self.dataset_path, sep=self.sep, header=None, names=self.columns_names, + dtype=self.dtype_dict) + if self.time: + self.pd_data = self.pd_data[self.columns_names] + self.preprocess_time() + # self.pd_data = col_func(self.pd_data) + else: + self.pd_data = self.pd_data[self.used_columns_names] + + def preprocessing(self, val_test_size, random_state): + if self.time: + self.X = self.pd_data.drop('i', axis=1).values + self.y = self.pd_data['i'].values + else: + self.X = self.pd_data.iloc[::, :-1].values + self.y = self.pd_data.iloc[::, -1].values + + self.user_num = max(self.X[::, 0]) + 1 + self.item_num = max(self.X[::, 1]) + 1 + self.train_X, self.val_test_X, self.train_y, self.val_test_y = train_test_split(self.X, self.y, + test_size=val_test_size * 2, + random_state=random_state) + # print(self.X) + # print(self.y) + # print(self.train_y) + # import time + # time.sleep(10) + + self.val_X, self.test_X, self.val_y, self.test_y = train_test_split(self.val_test_X, self.val_test_y, + test_size=0.5, + random_state=random_state) + + def preprocess_time(self): + self.pd_data['date'] = pd.to_datetime(self.pd_data['timestamp'].apply(datetime.utcfromtimestamp)) + self.pd_data['year'] = self.pd_data['date'].dt.year.astype(int) + self.pd_data['month'] = self.pd_data['date'].dt.month.astype(int) + self.pd_data['day'] = self.pd_data['date'].dt.day.astype(int) + # d.set_option('display.max_columns', None) + # pd.set_option('display.max_rows', None) + # print('start') + # print(self.pd_data.head(5)) + + # import time + # time.sleep(20) + + +class MovielensPreprocessor2(BaseRatingPredictionProprocessor): used_columns_names: List[str] def __init__(self, dataset_path, sep='::'): @@ -561,14 +624,15 @@ def _load_data(self): def preprocessing(self, val_test_size, random_state): self.X = self.pd_data.iloc[::, :-1].values - self.user_num = max( self.X[::,0] ) + 1 - self.item_num = max( self.X[::, 1] ) + 1 + self.user_num = max(self.X[::, 0]) + 1 + self.item_num = max(self.X[::, 1]) + 1 self.y = self.pd_data.iloc[::, -1].values - self.train_X, self.val_test_X, self.train_y, self.val_test_y = train_test_split(self.X, self.y, test_size = val_test_size * 2, - random_state=random_state) - self.val_X, self.test_X, self.val_y, self.test_y = train_test_split(self.val_test_X, self.val_test_y, test_size = 0.5, - random_state=random_state) - + self.train_X, self.val_test_X, self.train_y, self.val_test_y = train_test_split(self.X, self.y, + test_size=val_test_size * 2, + random_state=random_state) + self.val_X, self.test_X, self.val_y, self.test_y = train_test_split(self.val_test_X, self.val_test_y, + test_size=0.5, + random_state=random_state) class MovielensCTRPreprocessor(BasePointWiseProprocessor): @@ -602,11 +666,12 @@ def _negative_sampling(self, input_df, num_neg, mult=1): # Find sampled negative items (SNI) for each user u_sni_series = u_cni_series.agg( # series where index=user & data=sampled negative items - lambda x: np.random.RandomState().permutation(list(x)*num_neg*mult)[:num_neg * (len(item_set) - len(x))]) + lambda x: np.random.RandomState().permutation(list(x) * num_neg * mult)[ + :num_neg * (len(item_set) - len(x))]) # Distribute SNI to positive user-item interactions by chunk u_snic_series = u_sni_series.agg( # series where index=user & data=sampled negative item chunks (SNIC) - lambda x: [x[i*num_neg: (i+1)*num_neg] for i in range(int(len(x)/num_neg))]) + lambda x: [x[i * num_neg: (i + 1) * num_neg] for i in range(int(len(x) / num_neg))]) # Distribute SNIC to users u_snic_df = u_snic_series.to_frame().apply(pd.Series.explode).reset_index() @@ -621,7 +686,8 @@ def preprocessing(self, test_size, num_neg, random_state, mult): compact_X = compact_data.loc[:, compact_data.columns != 'rating'] compact_y = compact_data[['rating']] compact_train_X, compact_val_X, compact_train_y, compact_val_y = train_test_split(compact_X, compact_y, - test_size=test_size, random_state=random_state) + test_size=test_size, + random_state=random_state) def expand(unexpanded_X, unexpanded_y): # Extract positive relations diff --git a/autorecsys/recommender.py b/autorecsys/recommender.py index 325a78b..23a1f21 100644 --- a/autorecsys/recommender.py +++ b/autorecsys/recommender.py @@ -1,21 +1,21 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -from autorecsys.pipeline.graph import HyperGraph - -class RPRecommender(HyperGraph): - """A Rating-Prediction HyperModel based on connected Blocks and HyperBlocks. - # Arguments - inputs: A list of input node(s) for the HyperGraph. - outputs: A list of output node(s) for the HyperGraph. - """ - def __init__(self, **kwargs): - super().__init__(**kwargs) - -class CTRRecommender(HyperGraph): - """A CTR-Prediction HyperModel based on connected Blocks and HyperBlocks. - # Arguments - inputs: A list of input node(s) for the HyperGraph. - outputs: A list of output node(s) for the HyperGraph. - """ - def __init__(self, **kwargs): - super().__init__(**kwargs) +# from __future__ import absolute_import, division, print_function, unicode_literals +# +# from autorecsys.pipeline.graph import HyperGraph +# +# class RPRecommender(HyperGraph): +# """A Rating-Prediction HyperModel based on connected Blocks and HyperBlocks. +# # Arguments +# inputs: A list of input node(s) for the HyperGraph. +# outputs: A list of output node(s) for the HyperGraph. +# """ +# def __init__(self, **kwargs): +# super().__init__(**kwargs) +# +# class CTRRecommender(HyperGraph): +# """A CTR-Prediction HyperModel based on connected Blocks and HyperBlocks. +# # Arguments +# inputs: A list of input node(s) for the HyperGraph. +# outputs: A list of output node(s) for the HyperGraph. +# """ +# def __init__(self, **kwargs): +# super().__init__(**kwargs) diff --git a/autorecsys/searcher/__init__.py b/autorecsys/searcher/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/autorecsys/searcher/core/__init__.py b/autorecsys/searcher/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/autorecsys/searcher/core/hyperparameters.py b/autorecsys/searcher/core/hyperparameters.py deleted file mode 100755 index 33236e2..0000000 --- a/autorecsys/searcher/core/hyperparameters.py +++ /dev/null @@ -1,741 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - -import random -import math -import contextlib -import numpy as np -from tensorflow import keras -from autorecsys.searcher.core.utils import check_valid_params - -SUPPORT_DISTRIBUTION = {'loguniform', 'uniform'} - - -def _check_sampling_arg(sampling, - min_value, - max_value, - hp_type='int'): - if sampling is None: - return None - sampling = sampling.lower() - if sampling not in {'loguniform', 'uniform'}: - raise ValueError( - '`sampling` must be one of ' + str(SUPPORT_DISTRIBUTION)) - if sampling in {'loguniform'} and min_value <= 0: - raise ValueError( - '`sampling="' + str(sampling) + '" is not supported for ' - 'negative values, found `min_value`: ' + str(min_value)) - return sampling - - -class HyperParameter(object): - """HyperParameter base class. - - Args: - name: Str. Name of parameter. Must be unique. - default: Default value to return for the - parameter. - """ - def __init__(self, name, default=None): - self.name = name - self._default = default - - def get_config(self): - return {'name': self.name, 'default': self.default} - - @property - def default(self): - return self._default - - @classmethod - def from_config(cls, config): - return cls(**config) - - def random_sample(self, seed=None): - raise NotImplementedError - - -class Fixed(HyperParameter): - """Fixed, untunable value. - - # Arguments - name: Str. Name of parameter. Must be unique. - value: Value to use (can be any JSON-serializable - Python type). - """ - - def __init__(self, name, value): - self.name = name - self.value = value - - def __repr__(self): - return 'Fixed(name: {}, value: {})'.format( - self.name, self.value) - - def random_sample(self, seed=None): - return self.value - - @property - def default(self): - return self.value - - def get_config(self): - return {'name': self.name, 'value': self.value} - - -class Int(HyperParameter): - """Integer range. - - Args: - name: Str. Name of parameter. Must be unique. - min_value: Int. Lower limit of range (included). - max_value: Int. Upper limit of range (excluded). - sampling: Optional. One of "uniform", "loguniform", - Acts as a hint for an initial prior - probability distribution for how this value should - be sampled, e.g. "log" will assign equal - probabilities to each order of magnitude range. - default: Default value to return for the parameter. - If unspecified, the default value will be - `min_value`. - """ - - def __init__(self, - name, - min_value, - max_value, - sampling=None, - default=None): - super(Int, self).__init__(name=name, default=default) - self.max_value = check_valid_params(name=f'Int.max_value', x=max_value, - param_info={'type': 'int'}, skip_range_detect=True) - self.min_value = check_valid_params(name=f'Int.min_value', x=min_value, - param_info={'type': 'int'}, skip_range_detect=True) - self.sampling = _check_sampling_arg(sampling, min_value, max_value, hp_type='int') - - def __repr__(self): - return ('Int(name: "{}", min_value: {}, max_value: {}, ' - 'sampling: {}, default: {})').format( - self.name, - self.min_value, - self.max_value, - self.sampling, - self.default) - - def random_sample(self, seed=None): - random_state = random.Random(seed) - if self.sampling in {'loguniform', 'uniform'}: - cdf = float(random_state.random()) - if self.sampling == 'loguniform': - random_sample = _log_sample( - cdf, self.min_value, self.max_value) - else: - random_sample = _uniform_sample( - cdf, self.min_value, self.max_value) - return int(random_sample) - return int(random_state.uniform(self.min_value, self.max_value)) - - @property - def default(self): - if self._default is not None: - return self._default - return self.min_value - - def get_config(self): - config = super(Int, self).get_config() - config['min_value'] = self.min_value - config['max_value'] = self.max_value - config['sampling'] = self.sampling - config['default'] = self._default - return config - - -class Choice(HyperParameter): - """Choice of one value among a predefined set of possible values. - - Args: - name: Str. Name of parameter. Must be unique. - values: List of possible values. Values must be int, float, - str, or bool. All values must be of the same type. - ordered: Whether the values passed should be considered to - have an ordering. This defaults to `True` for float/int - values. Must be `False` for any other values. - default: Default value to return for the parameter. - If unspecified, the default value will be: - - None if None is one of the choices in `values` - - The first entry in `values` otherwise. - """ - - def __init__(self, name, values, ordered=None, default=None): - super(Choice, self).__init__(name=name, default=default) - if not values: - raise ValueError('`values` must be provided.') - self.values = values - - # Type checking. - types = set(type(v) for v in values) - unsupported_types = types - {int, float, str, bool} - if unsupported_types: - raise TypeError( - 'A `Choice` can contain only `int`, `float`, `str`, or ' - '`bool`, found values: ' + str(values) + 'with ' - 'types: ' + str(unsupported_types)) - - if len(types) > 1: - raise TypeError( - 'A `Choice` can contain only one type of value, found ' - 'values: ' + str(values) + ' with types ' + str(types)) - self._type = types.pop() - - # Get or infer ordered. - self.ordered = ordered - orderable_types = {int, float} - if self.ordered and self._type not in orderable_types: - raise ValueError('`ordered` must be `False` for non-numeric ' - 'types.') - if self.ordered is None: - self.ordered = self._type in orderable_types - - if default is not None and default not in values: - raise ValueError( - 'The default value should be one of the choices. ' - 'You passed: values=%s, default=%s' % (values, default)) - - def __repr__(self): - return 'Choice(name: "{}", values: {}, ordered: {}, default: {})'.format( - self.name, self.values, self.ordered, self.default) - - @property - def default(self): - if self._default is None: - if None in self.values: - return None - return self.values[0] - return self._default - - def random_sample(self, seed=None): - random_state = random.Random(seed) - return random_state.choice(self.values) - - def get_config(self): - config = super(Choice, self).get_config() - config['values'] = self.values - config['ordered'] = self.ordered - return config - - -class Float(HyperParameter): - """Floating point range, can be evenly divided. - - Args: - name: Str. Name of parameter. Must be unique. - min_value: Float. Lower bound of the range. - max_value: Float. Upper bound of the range. - sampling: Optional. One of "linear", "log", - "reverse_log". Acts as a hint for an initial prior - probability distribution for how this value should - be sampled, e.g. "log" will assign equal - probabilities to each order of magnitude range. - default: Default value to return for the parameter. - If unspecified, the default value will be - `min_value`. - """ - - def __init__(self, - name, - min_value, - max_value, - sampling=None, - default=None): - super(Float, self).__init__(name=name, default=default) - self.max_value = check_valid_params(name='Float.max_value', x=max_value, - param_info={'type': 'float'}, skip_range_detect=True) - self.min_value = check_valid_params(name='Float.min_value', x=min_value, - param_info={'type': 'float'}, skip_range_detect=True) - self.sampling = _check_sampling_arg(sampling, min_value, max_value, hp_type='float') - - def __repr__(self): - return ('Float(name: "{}", min_value: {}, max_value: {}, ' - 'sampling: {}, default: {})').format( - self.name, - self.min_value, - self.max_value, - self.sampling, - self.default) - - @property - def default(self): - if self._default is not None: - return self._default - return self.min_value - - def get_config(self): - config = super(Float, self).get_config() - config['min_value'] = self.min_value - config['max_value'] = self.max_value - config['sampling'] = self.sampling - return config - - def random_sample(self, seed=None): - random_state = random.Random(seed) - if self.sampling in {'loguniform', 'uniform'}: - cdf = float(random_state.random()) - if self.sampling == 'loguniform': - random_sample = _log_sample( - cdf, self.min_value, self.max_value) - else: - random_sample = _uniform_sample( - cdf, self.min_value, self.max_value) - return float(random_sample) - return random_state.uniform(self.min_value, self.max_value) - - -class Boolean(HyperParameter): - """Choice between True and False. - - Args: - name: Str. Name of parameter. Must be unique. - default: Default value to return for the parameter. - If unspecified, the default value will be False. - """ - - def __init__(self, name, default=False): - super(Boolean, self).__init__(name=name, default=default) - if default not in {True, False}: - raise ValueError( - '`default` must be a Python boolean. ' - 'You passed: default=%s' % (default,)) - - def __repr__(self): - return 'Boolean(name: "{}", default: {})'.format( - self.name, self.default) - - def random_sample(self, seed=None): - random_state = random.Random(seed) - return random_state.choice((True, False)) - - -class HyperParameters(object): - """Container for both a hyperparameter space, and current values. - - Attributes: - _space: A list of HyperParameter instances. - values: A dict mapping hyperparameter names to current values. - """ - - def __init__(self): - # A map from full HP name to HP object. - self._space = {} - self.values = {} - self._scopes = [] - - @contextlib.contextmanager - def name_scope(self, name): - self._scopes.append(name) - try: - yield - finally: - self._scopes.pop() - - @contextlib.contextmanager - def conditional_scope(self, parent_name, parent_values): - """Opens a scope to create conditional HyperParameters. - - All HyperParameters created under this scope will only be active - when the parent HyperParameter specified by `parent_name` is - equal to one of the values passed in `parent_values`. - - When the condition is not met, creating a HyperParameter under - this scope will register the HyperParameter, but will return - `None` rather than a concrete value. - - Note that any Python code under this scope will execute - regardless of whether the condition is met. - - Arguments: - parent_name: The name of the HyperParameter to condition on. - parent_values: Values of the parent HyperParameter for which - HyperParameters under this scope should be considered valid. - """ - full_parent_name = self._get_name(parent_name) - if full_parent_name not in self.values: - raise ValueError( - '`HyperParameter` named: ' + full_parent_name + ' ' - 'not defined.') - - if not isinstance(parent_values, (list, tuple)): - parent_values = [parent_values] - - parent_values = [str(v) for v in parent_values] - - self._scopes.append({'parent_name': parent_name, - 'parent_values': parent_values}) - try: - yield - finally: - self._scopes.pop() - - def _conditions_are_active(self, scopes=None): - if scopes is None: - scopes = self._scopes - - partial_scopes = [] - for scope in scopes: - if self._is_conditional_scope(scope): - full_name = self._get_name( - scope['parent_name'], - partial_scopes) - if str(self.values[full_name]) not in scope['parent_values']: - return False - partial_scopes.append(scope) - return True - - def _retrieve(self, - name, - type, - config, - parent_name=None, - parent_values=None, - overwrite=False): - """Gets or creates a `HyperParameter`.""" - if parent_name: - with self.conditional_scope(parent_name, parent_values): - return self._retrieve_helper(name, type, config, overwrite) - return self._retrieve_helper(name, type, config, overwrite) - - def _retrieve_helper(self, name, type, config, overwrite=False): - self._check_name_is_valid(name) - full_name = self._get_name(name) - - if full_name in self.values and not overwrite: - # TODO: type compatibility check, - # or name collision check. - retrieved_value = self.values[full_name] - else: - retrieved_value = self.register(name, type, config) - - if self._conditions_are_active(): - return retrieved_value - # Sanity check that a conditional HP that is not currently active - # is not being inadvertently relied upon in the model building - # function. - return None - - def register(self, name, type, config): - full_name = self._get_name(name) - config['name'] = full_name - config = {'class_name': type, 'config': config} - p = deserialize(config) - self._space[full_name] = p - value = p.default - self.values[full_name] = value - return value - - def get(self, name): - """Return the current value of this HyperParameter.""" - - # Most common case: not attempting to access a conditional parent - # or conditional child. - full_name = self._get_name(name) - if full_name in self.values: - if self._conditions_are_active(): - return self.values[full_name] - else: - # Sanity check for conditional HP usage. - return None - - # Check parent/child conditions. - found_inactive = False - # Remove conditional scopes from name. - full_name_no_cond = '/'.join([ - p for p in self._get_name_parts(full_name) if - not self._is_conditional_scope(p)]) - for hp_name in self.values.keys(): - hp_parts = self._get_name_parts(hp_name) - # Remove conditional scopes from name. - hp_name_no_cond = '/'.join( - [p for p in hp_parts - if not self._is_conditional_scope(p)]) - if hp_name_no_cond == full_name_no_cond: - # Check that this HP is active for this Trial. - if self._conditions_are_active(hp_parts): - return self.values[hp_name] - else: - found_inactive = True - - if found_inactive: - # Sanity check, found only inactive conditional HPs. - return None - - raise ValueError( - 'Unknown parameter: {}'.format(full_name_no_cond)) - - def Fixed(self, - name, - value, - parent_name=None, - parent_values=None): - return self._retrieve(name, 'Fixed', - config={'value': value}, - parent_name=parent_name, - parent_values=parent_values) - - def Choice(self, - name, - values, - ordered=None, - default=None, - parent_name=None, - parent_values=None): - return self._retrieve(name, 'Choice', - config={'values': values, - 'ordered': ordered, - 'default': default}, - parent_name=parent_name, - parent_values=parent_values) - - def Int(self, - name, - min_value, - max_value, - sampling=None, - default=None, - parent_name=None, - parent_values=None): - return self._retrieve(name, 'Int', - config={'min_value': min_value, - 'max_value': max_value, - 'sampling': sampling, - 'default': default}, - parent_name=parent_name, - parent_values=parent_values) - - def Float(self, - name, - min_value, - max_value, - sampling=None, - default=None, - parent_name=None, - parent_values=None): - return self._retrieve(name, 'Float', - config={'min_value': min_value, - 'max_value': max_value, - 'sampling': sampling, - 'default': default}, - parent_name=parent_name, - parent_values=parent_values) - - def Boolean(self, - name, - default=False, - parent_name=None, - parent_values=None): - return self._retrieve(name, 'Boolean', - config={'default': default}, - parent_name=parent_name, - parent_values=parent_values) - - @property - def space(self): - return list([hp for hp in self._space.values()]) - - def get_config(self): - return { - 'space': [{'class_name': p.__class__.__name__, - 'config': p.get_config()} - for p in self._space.values()], - 'values': dict((k, v) for (k, v) in self.values.items()), - } - - @classmethod - def from_config(cls, config): - hp = cls() - for p in config['space']: - p = deserialize(p) - hp._space[p.name] = p - hp.values = dict((k, v) for (k, v) in config['values'].items()) - return hp - - def copy(self): - return HyperParameters.from_config(self.get_config()) - - def merge(self, hps, overwrite=True): - """Merges hyperparameters into this object. - - Arguments: - hps: A `HyperParameters` object or list of `HyperParameter` - objects. - overwrite: bool. Whether existing `HyperParameter`s should - be overridden by those in `hps` with the same name. - """ - if isinstance(hps, HyperParameters): - hps = hps.space - for hp in hps: - self._retrieve( - hp.name, - hp.__class__.__name__, - hp.get_config(), - overwrite=overwrite) - - def _get_name(self, name, scopes=None): - """Returns a name qualified by `name_scopes`.""" - if scopes is None: - scopes = self._scopes - - scope_strings = [] - for scope in scopes: - if self._is_name_scope(scope): - scope_strings.append(scope) - elif self._is_conditional_scope(scope): - parent_name = scope['parent_name'] - parent_values = scope['parent_values'] - scope_string = '{name}={vals}'.format( - name=parent_name, - vals=','.join([str(val) for val in parent_values])) - scope_strings.append(scope_string) - return '/'.join(scope_strings + [name]) - - @staticmethod - def _get_name_parts(full_name): - """Splits `full_name` into its scopes and leaf name.""" - str_parts = full_name.split('/') - parts = [] - - for part in str_parts: - if '=' in part: - parent_name, parent_values = part.split('=') - parent_values = parent_values.split(',') - parts.append({'parent_name': parent_name, - 'parent_values': parent_values}) - else: - parts.append(part) - - return parts - - def _check_name_is_valid(self, name): - if '/' in name or '=' in name or ',' in name: - raise ValueError( - '`HyperParameter` names cannot contain "/", "=" or "," ' - 'characters.') - - for scope in self._scopes[::-1]: - if self._is_conditional_scope(scope): - if name == scope['parent_name']: - raise ValueError( - 'A conditional `HyperParameter` cannot have the same ' - 'name as its parent. Found: ' + str(name) + ' and ' - 'parent_name: ' + str(scope['parent_name'])) - else: - # Names only have to be unique up to the last `name_scope`. - break - - @staticmethod - def _is_name_scope(scope): - return isinstance(scope, str) - - @staticmethod - def _is_conditional_scope(scope): - return (isinstance(scope, dict) and - 'parent_name' in scope and 'parent_values' in scope) - - def get_value_in_nested_format(self): - - def helper(cur_dict_, lt_, val_): - if len(lt_) == 1: - cur_dict_.update({lt_[0]: val_}) - else: - if lt_[0] not in cur_dict_: - cur_dict_[lt_[0]] = dict() - helper(cur_dict_[lt_[0]], lt_[1:], val_) - - res = dict() - for key, val in self.values.items(): - lt = key.split('/') - helper(res, lt, val) - return res - - -def deserialize(config): - # Autograph messes with globals(), so in order to support HPs inside `call` we - # have to enumerate them manually here. - objects = [HyperParameter, Float, Int, Choice, Boolean, Fixed, HyperParameters] - module_objects = {cls.__name__: cls for cls in objects} - return keras.utils.deserialize_keras_object( - config, module_objects=module_objects) - - -def _log_sample(x, min_value, max_value): - """Applies log scale to a value in range [0, 1].""" - return min_value * math.pow(max_value / min_value, x) - -def _uniform_sample(x, min_value, max_value): - return min_value + (max_value - min_value) * x - -def cumulative_prob_to_value(prob, hp): - """Convert a value from [0, 1] to a hyperparameter value.""" - if isinstance(hp, Fixed): - return hp.value - elif isinstance(hp, Boolean): - return bool(prob >= 0.5) - elif isinstance(hp, Choice): - ele_prob = 1 / len(hp.values) - index = int(math.floor(prob / ele_prob)) - # Can happen when `prob` is very close to 1. - if index == len(hp.values): - index = index - 1 - return hp.values[index] - elif isinstance(hp, (Int, Float)): - sampling = hp.sampling or 'linear' - if sampling == 'linear': - value = prob * (hp.max_value - hp.min_value) + hp.min_value - elif sampling == 'log': - value = hp.min_value * math.pow(hp.max_value / hp.min_value, prob) - elif sampling == 'reverse_log': - value = (hp.max_value + hp.min_value - - hp.min_value * math.pow(hp.max_value / hp.min_value, 1 - prob)) - else: - raise ValueError('Unrecognized sampling value: {}'.format(sampling)) - - if hp.step is not None: - values = np.arange(hp.min_value, hp.max_value + 1e-7, step=hp.step) - closest_index = np.abs(values - value).argmin() - value = values[closest_index] - - if isinstance(hp, Int): - return int(value) - return value - else: - raise ValueError('Unrecognized HyperParameter type: {}'.format(hp)) - - -def value_to_cumulative_prob(value, hp): - """Convert a hyperparameter value to [0, 1].""" - if isinstance(hp, Fixed): - return 0.5 - if isinstance(hp, Boolean): - # Center the value in its probability bucket. - if value: - return 0.75 - return 0.25 - elif isinstance(hp, Choice): - ele_prob = 1 / len(hp.values) - index = hp.values.index(value) - # Center the value in its probability bucket. - return (index + 0.5) * ele_prob - elif isinstance(hp, (Int, Float)): - sampling = hp.sampling or 'linear' - if sampling == 'linear': - return (value - hp.min_value) / (hp.max_value - hp.min_value) - elif sampling == 'log': - return (math.log(value / hp.min_value) / - math.log(hp.max_value / hp.min_value)) - elif sampling == 'reverse_log': - return ( - 1. - math.log((hp.max_value + hp.min_value - value) / hp.min_value) / - math.log(hp.max_value / hp.min_value)) - else: - raise ValueError('Unrecognized sampling value: {}'.format(sampling)) - else: - raise ValueError('Unrecognized HyperParameter type: {}'.format(hp)) diff --git a/autorecsys/searcher/core/oracle.py b/autorecsys/searcher/core/oracle.py deleted file mode 100755 index 74e5229..0000000 --- a/autorecsys/searcher/core/oracle.py +++ /dev/null @@ -1,370 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - - -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -import glob -import collections -import json -import logging - -from autorecsys.searcher.core import trial as trial_lib, hyperparameters as hp_module -from autorecsys.searcher.core.trial import Stateful -from autorecsys.utils import metric -from autorecsys.utils.common import create_directory - -Objective = collections.namedtuple('Objective', 'name direction') -LOGGER = logging.getLogger(__name__) - - -class Oracle(Stateful): - """Implements a hyperparameter optimization algorithm. - - Attributes: - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_trials: The maximum number of hyperparameter - combinations to try. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. - """ - - def __init__(self, - objective, - max_trials=None, - hyperparameters=None, - allow_new_entries=True, - tune_new_entries=True): - self.objective = _format_objective(objective) - self.max_trials = max_trials - if not hyperparameters: - if not tune_new_entries: - raise ValueError( - 'If you set `tune_new_entries=False`, you must' - 'specify the search space via the ' - '`hyperparameters` argument.') - if not allow_new_entries: - raise ValueError( - 'If you set `allow_new_entries=False`, you must' - 'specify the search space via the ' - '`hyperparameters` argument.') - self.hyperparameters = hp_module.HyperParameters() - else: - self.hyperparameters = hyperparameters - self.allow_new_entries = allow_new_entries - self.tune_new_entries = tune_new_entries - - # trial_id -> Trial - self.trials = {} - # tuner_id -> Trial - self.ongoing_trials = {} - - # Set in `BaseTuner` via `set_project_dir`. - self._directory = None - self._project_name = None - - def _populate_space(self, trial_id): - """Fill the hyperparameter space with values for a trial. - - This method should be overrridden in subclasses and called in - `create_trial` in order to populate the hyperparameter space with - values. - - Args: - `trial_id`: The id for this Trial. - - Returns: - A dictionary with keys "values" and "status", where "values" is - a mapping of parameter names to suggested values, and "status" - is the TrialStatus that should be returned for this trial (one - of "RUNNING", "IDLE", or "STOPPED"). - """ - raise NotImplementedError - - def _score_trial(self, trial): - """Score a completed `Trial`. - - This method can be overridden in subclasses to provide a score for - a set of hyperparameter values. This method is called from `end_trial` - on completed `Trial`s. - - Args: - trial: A completed `Trial` object. - """ - # Assumes single objective, subclasses can override. - trial.score = trial.metrics.get_best_value(self.objective.name) - trial.best_step = trial.metrics.get_best_step(self.objective.name) - - def create_trial(self, tuner_id): - """Create a new `Trial` to be run by the `Tuner`. - - A `Trial` corresponds to a unique set of hyperparameters to be run - by `Tuner.run_trial`. - - Args: - tuner_id: A ID that identifies the `Tuner` requesting a - `Trial`. `Tuners` that should run the same trial (for instance, - when running a multi-worker model) should have the same ID. - - Returns: - A `Trial` object containing a set of hyperparameter values to run - in a `Tuner`. - """ - # Allow for multi-worker DistributionStrategy within a Trial. - if tuner_id in self.ongoing_trials: - return self.ongoing_trials[tuner_id] - - trial_id = trial_lib.generate_trial_id() - - if len(self.trials) >= self.max_trials: - status = trial_lib.TrialStatus.STOPPED - values = None - else: - response = self._populate_space(trial_id) - status = response['status'] - values = response['values'] if 'values' in response else None - - hyperparameters = self.hyperparameters.copy() - hyperparameters.values = values or {} - trial = trial_lib.Trial( - hyperparameters=hyperparameters, - trial_id=trial_id, - status=status) - - if status == trial_lib.TrialStatus.RUNNING: - self.ongoing_trials[tuner_id] = trial - self.trials[trial_id] = trial - self._save_trial(trial) - self.save() - - return trial - - def update_trial(self, trial_id, metrics, step=0): - """Used by a worker to report the status of a trial. - - Args: - trial_id: A previously seen trial id. - metrics: Dict of float. The current value of this - trial's metrics. - step: (Optional) Float. Used to report intermediate results. The - current value in a timeseries representing the state of the - trial. This is the value that `metrics` will be associated with. - - Returns: - Trial object. Trial.status will be set to "STOPPED" if the Trial - should be stopped early. - """ - trial = self.trials[trial_id] - self._check_objective_found(metrics) - for metric_name, metric_value in metrics.items(): - if not trial.metrics.exists(metric_name): - direction = _maybe_infer_direction_from_objective( - self.objective, metric_name) - trial.metrics.register(metric_name, direction=direction) - trial.metrics.update(metric_name, metric_value, step=step) - self._save_trial(trial) - # To signal early stopping, set Trial.status to "STOPPED". - return trial.status - - def end_trial(self, trial_id, status='COMPLETED'): - """Record the measured objective for a set of parameter values. - - Args: - trial_id: String. Unique id for this trial. - status: String, one of "COMPLETED", "INVALID". A status of - "INVALID" means a trial has crashed or been deemed - infeasible. - """ - trial = None - for tuner_id, ongoing_trial in self.ongoing_trials.items(): - if ongoing_trial.trial_id == trial_id: - trial = self.ongoing_trials.pop(tuner_id) - break - - if not trial: - raise ValueError( - 'Ongoing trial with id: {} not found.'.format(trial_id)) - - trial.status = status - if status == trial_lib.TrialStatus.COMPLETED: - self._score_trial(trial) - self._save_trial(trial) - self.save() - - def get_space(self): - """Returns the `HyperParameters` search space.""" - return self.hyperparameters.copy() - - def update_space(self, hyperparameters): - """Add new hyperparameters to the tracking space. - - Already recorded parameters get ignored. - - Args: - hyperparameters: An updated HyperParameters object. - """ - ref_names = {hp.name for hp in self.hyperparameters.space} - new_hps = [hp for hp in hyperparameters.space - if hp.name not in ref_names] - - if new_hps and not self.allow_new_entries: - raise RuntimeError('`allow_new_entries` is `False`, but found ' - 'new entries {}'.format(new_hps)) - - if not self.tune_new_entries: - # New entries should always use the default value. - return - - for hp in new_hps: - self.hyperparameters.register( - hp.name, hp.__class__.__name__, hp.get_config()) - - def get_trial(self, trial_id): - """Returns the `Trial` specified by `trial_id`.""" - return self.trials[trial_id] - - def get_best_trials(self, num_trials=1): - """Returns the best `Trial`s.""" - trials = [t for t in self.trials.values() - if t.status == trial_lib.TrialStatus.COMPLETED] - - sorted_trials = sorted( - trials, - key=lambda trial: trial.score, - # Assumes single objective, subclasses can override. - reverse=self.objective.direction == 'max' - ) - return sorted_trials[:num_trials] - - def remaining_trials(self): - if self.max_trials: - return self.max_trials - len(self.trials.items()) - else: - return None - - def get_state(self): - # `self.trials` are saved in their own, Oracle-agnostic files. - # Just save the IDs for ongoing trials, since these are in `trials`. - state = {} - state['ongoing_trials'] = { - tuner_id: trial.trial_id - for tuner_id, trial in self.ongoing_trials.items()} - # Hyperparameters are part of the state because they can be added to - # during the course of the search. - state['hyperparameters'] = self.hyperparameters.get_config() - return state - - def set_state(self, state): - # `self.trials` are saved in their own, Oracle-agnostic files. - self.ongoing_trials = { - tuner_id: self.trials[trial_id] - for tuner_id, trial_id in state['ongoing_trials'].items()} - self.hyperparameters = hp_module.HyperParameters.from_config( - state['hyperparameters']) - - def set_project_dir(self, directory, project_name, overwrite=False): - """Sets the project directory and reloads the Oracle.""" - self._directory = directory - self._project_name = project_name - if (not overwrite) and os.path.exists(self._get_oracle_fname()): - LOGGER.info('Reloading Oracle from {}'.format( - self._get_oracle_fname())) - self.reload() - - @property - def _project_dir(self): - dirname = os.path.join( - self._directory, - self._project_name) - create_directory(dirname) - return dirname - - def save(self): - # `self.trials` are saved in their own, Oracle-agnostic files. - super(Oracle, self).save(self._get_oracle_fname()) - - def reload(self): - # Reload trials from their own files. - trial_dirs = glob.glob(os.path.join(self._project_dir, 'trial_*')) - trial_fnames = [os.path.join(trial_dir, 'trial.json') for trial_dir in trial_dirs] - for fname in trial_fnames: - with open(fname, 'r') as fp: - trial_state = json.load(fp) - trial = trial_lib.Trial.from_state(trial_state) - self.trials[trial.trial_id] = trial - super(Oracle, self).reload(self._get_oracle_fname()) - - def _get_oracle_fname(self): - return os.path.join( - self._project_dir, - 'oracle.json') - - @staticmethod - def _compute_values_hash(values): - keys = sorted(values.keys()) - s = ''.join(str(k) + '=' + str(values[k]) for k in keys) - # return hashlib.sha256(s.encode('utf-8')).hexdigest()[:32] - return hash(s) - - def _check_objective_found(self, metrics): - if isinstance(self.objective, Objective): - objective_names = [self.objective.name] - else: - objective_names = [obj.name for obj in self.objective] - for metric_name in metrics.keys(): - if metric_name in objective_names: - objective_names.remove(metric_name) - if objective_names: - raise ValueError( - 'Objective value missing in metrics reported to the ' - 'Oracle, expected: {}, found: {}'.format( - objective_names, metrics.keys())) - - def _get_trial_dir(self, trial_id): - dirname = os.path.join( - self._project_dir, - 'trial_' + str(trial_id)) - create_directory(dirname) - return dirname - - def _save_trial(self, trial): - # Write trial status to trial directory - trial_id = trial.trial_id - trial.save(os.path.join( - self._get_trial_dir(trial_id), - 'trial.json')) - - -def _format_objective(objective): - if isinstance(objective, list): - return [_format_objective(obj) for obj in objective] - if isinstance(objective, Objective): - return objective - if isinstance(objective, str): - direction = metric.infer_metric_direction(objective) - return Objective(name=objective, direction=direction) - else: - raise ValueError('`objective` not understood, expected str or ' - '`Objective` object, found: {}'.format(objective)) - - -def _maybe_infer_direction_from_objective(objective, metric_name): - if isinstance(objective, Objective): - objective = [objective] - for obj in objective: - if obj.name == metric_name: - return obj.direction - return None diff --git a/autorecsys/searcher/core/trial.py b/autorecsys/searcher/core/trial.py deleted file mode 100755 index 514d438..0000000 --- a/autorecsys/searcher/core/trial.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - - -from __future__ import absolute_import, division, print_function, unicode_literals - -import random -import tensorflow as tf -import time -import json - -from autorecsys.searcher.core import hyperparameters as hp_module -from autorecsys.utils import display, metric - - -class Stateful(object): - - def get_state(self): - raise NotImplementedError - - def set_state(self, state): - raise NotImplementedError - - def save(self, fname): - state = self.get_state() - state_json = json.dumps(state) - with open(fname, 'w') as fp: - fp.write(state_json) - return str(fname) - - def reload(self, fname): - with open(fname, 'r') as fp: - state = json.load(fp) - self.set_state(state) - - -class TrialStatus: - RUNNING = 'RUNNING' - IDLE = 'IDLE' - INVALID = 'INVALID' - STOPPED = 'STOPPED' - COMPLETED = 'COMPLETED' - - -class Trial(Stateful): - - def __init__(self, - hyperparameters, - trial_id=None, - status=TrialStatus.RUNNING): - self.hyperparameters = hyperparameters - self.trial_id = generate_trial_id() if trial_id is None else trial_id - self.metrics = metric.MetricsTracker() - self.score = None - self.best_step = None - self.status = status - - def summary(self): - display.section('Trial summary') - if self.hyperparameters.values: - display.subsection('Hp values:') - value_need_display = {k: v for k, v in self.hyperparameters.values.items() - if k in self.hyperparameters._space and - self.hyperparameters._space[k].__class__.__name__ != 'Fixed'} - display.display_settings(value_need_display) - else: - display.subsection('Hp values: default configuration.') - if self.score is not None: - display.display_setting('Score: {}'.format(self.score)) - if self.best_step is not None: - display.display_setting('Best step: {}'.format(self.best_step)) - - def get_state(self): - return { - 'trial_id': self.trial_id, - 'hyperparameters': self.hyperparameters.get_config(), - 'metrics': self.metrics.get_config(), - 'score': self.score, - 'best_step': self.best_step, - 'status': self.status - } - - def set_state(self, state): - self.trial_id = state['trial_id'] - hp = hp_module.HyperParameters.from_config( - state['hyperparameters'] - ) - self.hyperparameters = hp - self.metrics = metric.MetricsTracker.from_config(state['metrics']) - self.score = state['score'] - self.best_step = state['best_step'] - self.status = state['status'] - - @classmethod - def from_state(cls, state): - trial = cls(hyperparameters=None) - trial.set_state(state) - return trial - - @classmethod - def load(cls, fname): - with tf.io.gfile.GFile(fname, 'r') as f: - state_data = f.read() - return cls.from_state(state_data) - - -def generate_trial_id(): - s = str(time.time()) + str(random.randint(1, 1e7)) - # return hashlib.sha256(s.encode('utf-8')).hexdigest()[:32] - return hash(s) % 1045543567 diff --git a/autorecsys/searcher/core/utils.py b/autorecsys/searcher/core/utils.py deleted file mode 100644 index 388629a..0000000 --- a/autorecsys/searcher/core/utils.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging - -LOGGER = logging.getLogger(__name__) -TYPE_MAP = {'int': int, 'float': float, 'str': str, 'list': list, 'tuple': tuple, 'bool': bool} -CANT_BE_SET = -1 - - -def check_valid_params(name, x, param_info, skip_range_detect): - param_type = TYPE_MAP[param_info['type']] - try: - x = param_type(x) - except ValueError as e: - LOGGER.exception(f'can not cast {name} to {param_type}') - raise e - param_range = param_info.get('range', None) - if param_range == CANT_BE_SET: - raise TypeError(f'{name} can not be set from config files') - if not skip_range_detect: - if isinstance(param_range, tuple): - if x not in param_range: - raise ValueError(f'{name} must be in {param_range}, {x} doesn\'t') - elif isinstance(param_range, list): - low, high = param_range - if x < low or x > high: - raise ValueError(f'{name} valid range: x>={low} && x<={high}') - else: - raise NotImplementedError(f'code error: the param\'range of a model must be tuple, list') - return x diff --git a/autorecsys/searcher/tuners/__init__.py b/autorecsys/searcher/tuners/__init__.py deleted file mode 100644 index fd746cf..0000000 --- a/autorecsys/searcher/tuners/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from .randomsearch import RandomSearch -from .bayesian import BayesianOptimization -from .greedy import Greedy - -TUNER_CLASSES = { - 'random': RandomSearch, - 'bayesian': BayesianOptimization, - "greedy": Greedy -} - - -def get_tuner_class(tuner): - if isinstance(tuner, str) and tuner in TUNER_CLASSES: - return TUNER_CLASSES.get(tuner) - else: - raise ValueError('The value {tuner} passed for argument tuner is invalid, ' - 'expected one of "random","bayesian".'.format(tuner=tuner)) diff --git a/autorecsys/searcher/tuners/bayesian.py b/autorecsys/searcher/tuners/bayesian.py deleted file mode 100644 index 6433d9f..0000000 --- a/autorecsys/searcher/tuners/bayesian.py +++ /dev/null @@ -1,339 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - - -import random - -import numpy as np -from scipy import optimize as scipy_optimize -from sklearn import exceptions -from sklearn import gaussian_process - -from autorecsys.searcher.core import hyperparameters as hp_module -from autorecsys.searcher.core import trial as trial_lib -from autorecsys.searcher.tuners.tuner import PipeTuner -from autorecsys.searcher.core import oracle as oracle_module - - -class BayesianOptimizationOracle(oracle_module.Oracle): - """Bayesian optimization oracle. - - It uses Bayesian optimization with a underlying Gaussian process model. - The acquisition function used is upper confidence bound (UCB), which can - be found in the following link: - https://www.cse.wustl.edu/~garnett/cse515t/spring_2015/files/lecture_notes/12.pdf - - # Arguments - objective: String or `kerastuner.Objective`. If a string, - the direction of the optimization (min or max) will be - inferred. - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested if the search space has been - exhausted. - num_initial_points: (Optional) Int. The number of randomly generated samples - as initial training data for Bayesian optimization. If not specified, - a value of 3 times the dimensionality of the hyperparameter space is - used. - alpha: Float. Value added to the diagonal of the kernel matrix - during fitting. It represents the expected amount of noise - in the observed performances in Bayesian optimization. - beta: Float. The balancing factor of exploration and exploitation. - The larger it is, the more explorative it is. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. - """ - - def __init__(self, - objective, - max_trials, - num_initial_points=None, - alpha=1e-4, - beta=2.6, - seed=None, - hyperparameters=None, - allow_new_entries=True, - tune_new_entries=True): - super(BayesianOptimizationOracle, self).__init__( - objective=objective, - max_trials=max_trials, - hyperparameters=hyperparameters, - tune_new_entries=tune_new_entries, - allow_new_entries=allow_new_entries) - self.num_initial_points = num_initial_points - self.alpha = alpha - self.beta = beta - self.seed = seed or random.randint(1, 1e4) - self._seed_state = self.seed - self._tried_so_far = set() - self._max_collisions = 20 - self._random_state = np.random.RandomState(self.seed) - self.gpr = self._make_gpr() - - def _make_gpr(self): - return gaussian_process.GaussianProcessRegressor( - kernel=gaussian_process.kernels.Matern(nu=2.5), - n_restarts_optimizer=20, - normalize_y=True, - alpha=self.alpha, - random_state=self.seed) - - def _populate_space(self, trial_id): - # Generate enough samples before training Gaussian process. - completed_trials = [t for t in self.trials.values() - if t.status == 'COMPLETED'] - - # Use 3 times the dimensionality of the space as the default number of - # random points. - dimensions = len(self.hyperparameters.space) - num_initial_points = self.num_initial_points or 3 * dimensions - if len(completed_trials) < num_initial_points: - values = self._random_trial() - return {'status': trial_lib.TrialStatus.RUNNING, - 'values': values} - - # Fit a GPR to the completed trials and return the predicted optimum values. - x, y = self._vectorize_trials() - try: - self.gpr.fit(x, y) - except exceptions.ConvergenceWarning: - # If convergence of the GPR fails, create a random trial. - values = self._random_trial() - return {'status': trial_lib.TrialStatus.RUNNING, - 'values': values} - - def _upper_confidence_bound(x): - x = x.reshape(1, -1) - mu, sigma = self.gpr.predict(x, return_std=True) - return mu - self.beta * sigma - - optimal_val = float('inf') - optimal_x = None - num_restarts = 50 - bounds = self._get_hp_bounds() - x_seeds = self._random_state.uniform(bounds[:, 0], bounds[:, 1], - size=(num_restarts, bounds.shape[0])) - for x_try in x_seeds: - # Sign of score is flipped when maximizing. - result = scipy_optimize.minimize(_upper_confidence_bound, - x0=x_try, - bounds=bounds, - method='L-BFGS-B') - if result.fun[0] < optimal_val: - optimal_val = result.fun[0] - optimal_x = result.x - - values = self._vector_to_values(optimal_x) - return {'status': trial_lib.TrialStatus.RUNNING, - 'values': values} - - def get_state(self): - state = super(BayesianOptimizationOracle, self).get_state() - state.update({ - 'num_initial_points': self.num_initial_points, - 'alpha': self.alpha, - 'beta': self.beta, - 'seed': self.seed, - 'seed_state': self._seed_state, - 'tried_so_far': list(self._tried_so_far), - 'max_collisions': self._max_collisions, - }) - return state - - def set_state(self, state): - super(BayesianOptimizationOracle, self).set_state(state) - self.num_initial_points = state['num_initial_points'] - self.alpha = state['alpha'] - self.beta = state['beta'] - self.seed = state['seed'] - self._seed_state = state['seed_state'] - self._tried_so_far = set(state['tried_so_far']) - self._max_collisions = state['max_collisions'] - self.gpr = self._make_gpr() - - def _random_trial(self): - """Fill a given hyperparameter space with values. - - Returns: - A dictionary mapping parameter names to suggested values. - Note that if the Oracle is keeping tracking of a large - space, it may return values for more parameters - than what was listed in `space`. - """ - collisions = 0 - while 1: - # Generate a set of random values. - values = {} - for p in self.hyperparameters.space: - values[p.name] = p.random_sample(self._seed_state) - self._seed_state += 1 - # Keep trying until the set of values is unique, - # or until we exit due to too many collisions. - values_hash = self._compute_values_hash(values) - if values_hash in self._tried_so_far: - collisions += 1 - if collisions > self._max_collisions: - return None - continue - self._tried_so_far.add(values_hash) - break - return values - - def _vectorize_trials(self): - x = [] - y = [] - ongoing_trials = {t for t in self.ongoing_trials.values()} - for trial in self.trials.values(): - # Create a vector representation of each Trial's hyperparameters. - trial_values = trial.hyperparameters.values - vector = [] - for hp in self._nonfixed_space(): - # Hyperparameters could have been added to the study since - # the trial was run. - if hp.name in trial_values: - trial_value = trial_values[hp.name] - else: - trial_value = hp.default - - # Embed an HP value into the continuous space [0, 1]. - prob = hp_module.value_to_cumulative_prob(trial_value, hp) - vector.append(prob) - - if trial in ongoing_trials: - # "Hallucinate" the results of ongoing trials. This ensures that - # repeat trials are not selected when running distributed. - x_h = np.array(vector).reshape((1, -1)) - y_h_mean, y_h_std = self.gpr.predict(x_h, return_std=True) - # Give a pessimistic estimate of the ongoing trial. - score = y_h_mean[0] + y_h_std[0] - elif trial.status == 'COMPLETED': - score = trial.score - # Always frame the optimization as a minimization for scipy.minimize. - if self.objective.direction == 'max': - score = -1 * score - else: - continue - - x.append(vector) - y.append(score) - - x = np.array(x) - y = np.array(y) - return x, y - - def _vector_to_values(self, vector): - values = {} - vector_index = 0 - for index, hp in enumerate(self.hyperparameters.space): - hp = self.hyperparameters.space[index] - if isinstance(hp, hp_module.Fixed): - values[hp.name] = hp.value - continue - - prob = vector[vector_index] - vector_index += 1 - - values[hp.name] = hp_module.cumulative_prob_to_value(prob, hp) - - return values - - def _find_closest(self, val, hp): - values = [hp.min_value] - while values[-1] + hp.step <= hp.max_value: - values.append(values[-1] + hp.step) - - array = np.asarray(values) - index = (np.abs(values - val)).argmin() - return array[index] - - def _get_hp_index(self, name): - for index, hp in enumerate(self.hyperparameters.space): - if hp.name == name: - return index - return None - - def _nonfixed_space(self): - return [hp for hp in self.hyperparameters.space - if not isinstance(hp, hp_module.Fixed)] - - def _get_hp_bounds(self): - bounds = [] - for hp in self._nonfixed_space(): - bounds.append([0, 1]) - return np.array(bounds) - - -class BayesianOptimization(PipeTuner): - """BayesianOptimization tuning with Gaussian process. - - # Arguments: - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested if the search space has - been exhausted. - num_initial_points: Int. The number of randomly generated samples as initial - training data for Bayesian optimization. - alpha: Float or array-like. Value added to the diagonal of - the kernel matrix during fitting. - beta: Float. The balancing factor of exploration and exploitation. - The larger it is, the more explorative it is. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. - **kwargs: Keyword arguments relevant to all `Tuner` subclasses. - Please see the docstring for `Tuner`. - """ - - def __init__(self, - hypergraph, - objective, - max_trials, - num_initial_points=2, - seed=None, - hyperparameters=None, - tune_new_entries=True, - allow_new_entries=True, - **kwargs): - oracle = BayesianOptimizationOracle(objective=objective, - max_trials=max_trials, - num_initial_points=num_initial_points, - seed=seed, - hyperparameters=hyperparameters, - tune_new_entries=tune_new_entries, - allow_new_entries=allow_new_entries) - super(BayesianOptimization, self, ).__init__(oracle, - hypergraph, - **kwargs) - - @classmethod - def get_name(cls): - return 'bayesian' diff --git a/autorecsys/searcher/tuners/greedy.py b/autorecsys/searcher/tuners/greedy.py deleted file mode 100644 index d0458a0..0000000 --- a/autorecsys/searcher/tuners/greedy.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - - -from __future__ import absolute_import, division, print_function, unicode_literals - -import random -import numpy as np - -from autorecsys.searcher.tuners.tuner import PipeTuner -from autorecsys.searcher.core import hyperparameters as hp_module -from autorecsys.searcher.core import oracle as oracle_module -from autorecsys.searcher.core import trial as trial_lib - - -class GreedyOracle(oracle_module.Oracle): - """An oracle combining random search and greedy algorithm. - It groups the HyperParameters into several categories, namely, HyperGraph, - Preprocessor, Architecture, and Optimization. The oracle tunes each group - separately using random search. In each trial, it use a greedy strategy to - generate new values for one of the categories of HyperParameters and use the best - trial so far for the rest of the HyperParameters values. - # Arguments - initial_hps: A list of dictionaries in the form of - {HyperParameter name (String): HyperParameter value}. - Each dictionary is one set of HyperParameters, which are used as the - initial trials for the search. Defaults to None. - seed: Int. Random seed. - """ - - HYPER = 'HYPER' - PREPROCESS = 'PREPROCESS' - OPT = 'OPT' - ARCH = 'ARCH' - STAGES = [HYPER, PREPROCESS, OPT, ARCH] - - @staticmethod - def next_stage(stage): - stages = GreedyOracle.STAGES - return stages[(stages.index(stage) + 1) % len(stages)] - - def __init__(self, - hypermodel, - initial_hps=None, - seed=None, - **kwargs): - super().__init__(**kwargs) - self.initial_hps = initial_hps or [] - self._tried_initial_hps = [False] * len(self.initial_hps) - self.hypermodel = hypermodel - # Sets of HyperParameter names. - self._hp_names = { - GreedyOracle.HYPER: set(), - GreedyOracle.PREPROCESS: set(), - GreedyOracle.OPT: set(), - GreedyOracle.ARCH: set(), - } - # The quota used to tune each category of hps. - self.seed = seed or random.randint(1, 1e4) - # Incremented at every call to `populate_space`. - self._seed_state = self.seed - self._tried_so_far = set() - self._max_collisions = 5 - - def update_space(self, hyperparameters): - # Get the block names. - self.hypermodel.build(hyperparameters) - - # Add the new Hyperparameters to different categories. - ref_names = {hp.name for hp in self.hyperparameters.space} - for hp in hyperparameters.space: - if hp.name not in ref_names: - hp_type = GreedyOracle.ARCH - self._hp_names[hp_type].add(hp.name) - super().update_space(hyperparameters) - - def _generate_stage(self): - probabilities = np.array([pow(len(value), 2) - for value in self._hp_names.values()]) - sum_p = np.sum(probabilities) - if sum_p == 0: - probabilities = np.array([1] * len(probabilities)) - sum_p = np.sum(probabilities) - probabilities = probabilities / sum_p - return np.random.choice(list(self._hp_names.keys()), p=probabilities) - - def _next_initial_hps(self): - for index, hps in enumerate(self.initial_hps): - if not self._tried_initial_hps[index]: - self._tried_initial_hps[index] = True - return hps - - def _populate_space(self, trial_id): - if not all(self._tried_initial_hps): - return {'status': trial_lib.TrialStatus.RUNNING, - 'values': self._next_initial_hps()} - - stage = self._generate_stage() - for _ in range(len(GreedyOracle.STAGES)): - values = self._generate_stage_values(stage) - # Reached max collisions. - if values is None: - # Try next stage. - stage = GreedyOracle.next_stage(stage) - continue - # Values found. - return {'status': trial_lib.TrialStatus.RUNNING, - 'values': values} - # All stages reached max collisions. - return {'status': trial_lib.TrialStatus.STOPPED, - 'values': None} - - def _generate_stage_values(self, stage): - best_trials = self.get_best_trials() - if best_trials: - best_values = best_trials[0].hyperparameters.values - else: - best_values = self.hyperparameters.values - collisions = 0 - while True: - # Generate new values for the current stage. - values = {} - for p in self.hyperparameters.space: - if p.name in self._hp_names[stage]: - values[p.name] = p.random_sample(self._seed_state) - self._seed_state += 1 - values = {**best_values, **values} - # Keep trying until the set of values is unique, - # or until we exit due to too many collisions. - values_hash = self._compute_values_hash(values) - if values_hash not in self._tried_so_far: - self._tried_so_far.add(values_hash) - break - collisions += 1 - if collisions > self._max_collisions: - # Reached max collisions. No value to return. - return None - return values - - -class Greedy(PipeTuner): - - def __init__(self, - hypergraph, - objective, - max_trials, - initial_hps=None, - seed=None, - hyperparameters=None, - tune_new_entries=True, - allow_new_entries=True, - **kwargs): - self.seed = seed - oracle = GreedyOracle(hypermodel=hypergraph, - objective=objective, - max_trials=max_trials, - initial_hps=initial_hps, - seed=seed, - hyperparameters=hyperparameters, - tune_new_entries=tune_new_entries, - allow_new_entries=allow_new_entries) - super(Greedy, self).__init__(oracle, - hypergraph, - **kwargs) - - @classmethod - def get_name(cls): - return 'greedy' diff --git a/autorecsys/searcher/tuners/randomsearch.py b/autorecsys/searcher/tuners/randomsearch.py deleted file mode 100644 index 799eb74..0000000 --- a/autorecsys/searcher/tuners/randomsearch.py +++ /dev/null @@ -1,164 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - - -"Basic random search searcher." - -from __future__ import absolute_import, division, print_function, unicode_literals - -import random - -from autorecsys.searcher.tuners.tuner import PipeTuner -from autorecsys.searcher.core import hyperparameters as hp_module -from autorecsys.searcher.core import oracle as oracle_module -from autorecsys.searcher.core import trial as trial_lib - - -class RandomSearchOracle(oracle_module.Oracle): - """Random search oracle. - Attributes: - objective: String or `kerastuner.Objective`. If a string, - the direction of the optimization (min or max) will be - inferred. - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. - """ - - def __init__(self, - objective, - max_trials, - seed=None, - hyperparameters=None, - allow_new_entries=True, - tune_new_entries=True): - super(RandomSearchOracle, self).__init__( - objective=objective, - max_trials=max_trials, - hyperparameters=hyperparameters, - tune_new_entries=tune_new_entries, - allow_new_entries=allow_new_entries) - self.seed = seed or random.randint(1, 1e4) - # Incremented at every call to `populate_space`. - self._seed_state = self.seed - # Hashes of values tried so far. - self._tried_so_far = set() - # Maximum number of identical values that can be generated - # before we consider the space to be exhausted. - self._max_collisions = 5 - - def _populate_space(self, _): - """Fill the hyperparameter space with values. - Args: - `trial_id`: The id for this Trial. - Returns: - A dictionary with keys "values" and "status", where "values" is - a mapping of parameter names to suggested values, and "status" - is the TrialStatus that should be returned for this trial (one - of "RUNNING", "IDLE", or "STOPPED"). - """ - collisions = 0 - while 1: - # Generate a set of random values. - values = {} - if all(isinstance(p, hp_module.Fixed) for p in self.hyperparameters.space): - break - for p in self.hyperparameters.space: - values[p.name] = p.random_sample(self._seed_state) - self._seed_state += 1 - # Keep trying until the set of values is unique, - # or until we exit due to too many collisions. - values_hash = self._compute_values_hash(values) - if values_hash in self._tried_so_far: - collisions += 1 - if collisions > self._max_collisions: - return {'status': trial_lib.TrialStatus.STOPPED, - 'values': None} - continue - self._tried_so_far.add(values_hash) - break - return {'status': trial_lib.TrialStatus.RUNNING, - 'values': values} - - def get_state(self): - state = super(RandomSearchOracle, self).get_state() - state.update({ - 'seed': self.seed, - 'seed_state': self._seed_state, - 'tried_so_far': list(self._tried_so_far), - }) - return state - - def set_state(self, state): - super(RandomSearchOracle, self).set_state(state) - self.seed = state['seed'] - self._seed_state = state['seed_state'] - self._tried_so_far = set(state['tried_so_far']) - - -class RandomSearch(PipeTuner): - """Random search tuner. - # Arguments: - config: Dictionary. Specify the search configurations - including TrainOptions, ModelOptions, Search Options. - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. - **kwargs: Keyword arguments relevant to all `Tuner` subclasses. - Please see the docstring for `Tuner`. - """ - - def __init__(self, - hypergraph, - objective, - max_trials, - seed=None, - hyperparameters=None, - tune_new_entries=True, - allow_new_entries=True, - **kwargs): - self.seed = seed - oracle = RandomSearchOracle(objective=objective, - max_trials=max_trials, - seed=seed, - hyperparameters=hyperparameters, - tune_new_entries=tune_new_entries, - allow_new_entries=allow_new_entries) - super(RandomSearch, self).__init__(oracle, - hypergraph, - **kwargs) - - @classmethod - def get_name(cls): - return 'random' diff --git a/autorecsys/searcher/tuners/tuner.py b/autorecsys/searcher/tuners/tuner.py deleted file mode 100755 index a287837..0000000 --- a/autorecsys/searcher/tuners/tuner.py +++ /dev/null @@ -1,742 +0,0 @@ -# -*- coding: utf-8 -*- -# This codes are migrated from Keras Tuner: https://keras-team.github.io/keras-tuner/. -# The copyright belows to the Keras Tuner authors. - - -"Tuner base class." - -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -import copy -import inspect -import shutil -import logging -import collections - -import tensorflow as tf -import numpy as np -from autorecsys.utils import display -from autorecsys.utils.common import create_directory -from autorecsys.searcher.core import trial as trial_module -from autorecsys.searcher.core import oracle as oracle_module -# from autorecsys.searcher.tuners import RandomSearch -# from autorecsys.searcher.tuners.hyperband import Hyperband - -# from sklearn.metrics import roc_auc_score, log_loss, mean_squared_error -# METRIC = {'auc': roc_auc_score, 'log_loss': log_loss, 'mse': mean_squared_error} - -METRIC = {'auc': tf.keras.metrics.AUC(), - 'logloss': tf.keras.metrics.CategoricalAccuracy(), - 'mse': tf.keras.metrics.MeanSquaredError(), - 'mae': tf.keras.metrics.MeanAbsoluteError(), - 'BinaryCrossentropy': tf.keras.metrics.BinaryCrossentropy(), } - -# TODO: Add more extensive display. -LOGGER = logging.getLogger(__name__) - - -class Display(object): - - @staticmethod - def on_trial_begin(trial): - display.section('New model') - trial.summary() - - @staticmethod - def on_trial_end(trial): - display.section('Trial complete') - trial.summary() - - -class TunerCallback(tf.keras.callbacks.Callback): - - def __init__(self, tuner, trial): - self.tuner = tuner - self.trial = trial - - def on_epoch_begin(self, epoch, logs=None): - self.tuner.on_epoch_begin( - self.trial, self.model, epoch, logs=logs) - - def on_batch_begin(self, batch, logs=None): - self.tuner.on_batch_begin(self.trial, self.model, batch, logs) - - def on_batch_end(self, batch, logs=None): - self.tuner.on_batch_end(self.trial, self.model, batch, logs) - - def on_epoch_end(self, epoch, logs=None): - self.tuner.on_epoch_end( - self.trial, self.model, epoch, logs=logs) - - -class BaseTuner(trial_module.Stateful): - """Tuner base class. - May be subclassed to create new tuners, including for non-Keras models. - Args: - oracle: Instance of Oracle class. - directory: String. Path to the working directory (relative). - project_name: Name to use as prefix for files saved - by this Tuner. - tuner_id: Optional. Used only with multi-worker DistributionStrategies. - overwrite: Bool, default `False`. If `False`, reloads an existing project - of the same name if one is found. Otherwise, overwrites the project. - """ - - def __init__(self, - oracle, - directory=None, - project_name=None, - logger=None, - overwrite=False): - # Ops and metadata - self.directory = directory or '.' - self.project_name = project_name or 'untitled_project' - if overwrite and os.path.exists(self.project_dir): - shutil.rmtree(self.project_dir) - - if not isinstance(oracle, oracle_module.Oracle): - raise ValueError('Expected oracle to be ' - 'an instance of Oracle, got: %s' % (oracle,)) - self.oracle = oracle - self.oracle.set_project_dir(self.directory, self.project_name, overwrite=overwrite) - - # To support tuning distribution. - self.tuner_id = os.environ.get('KERASTUNER_TUNER_ID', 'tuner0') - - # Logs etc - self.logger = logger - self._display = Display() - - # Populate initial search space. - hp = self.oracle.get_space() - self.oracle.update_space(hp) - - if not overwrite and os.path.exists(self._get_tuner_fname()): - LOGGER.info('Reloading Tuner from {}'.format( - self._get_tuner_fname())) - self.reload() - - def search(self, *fit_args, **fit_kwargs): - """Performs a search for best hyperparameter configuations. - # Arguments: - *fit_args: Positional arguments that should be passed to - `run_trial`, for example the training and validation data. - *fit_kwargs: Keyword arguments that should be passed to - `run_trial`, for example the training and validation data. - """ - self.on_search_begin() - while True: - trial = self.oracle.create_trial(self.tuner_id) - if trial.status == trial_module.TrialStatus.STOPPED: - # Oracle triggered exit. - tf.get_logger().info('Oracle triggered exit') - break - if trial.status == trial_module.TrialStatus.IDLE: - # Oracle is calculating, resend request. - continue - - self.on_trial_begin(trial) - model = self.run_trial(trial, *fit_args, **fit_kwargs) - self.on_trial_end(trial, model) - self.on_search_end() - - def run_trial(self, trial, *fit_args, **fit_kwargs): - """Evaluates a set of hyperparameter values. - This method is called during `search` to evaluate a set of - hyperparameters. - For subclass implementers: This method is responsible for - reporting metrics related to the `Trial` to the `Oracle` - via `self.oracle.update_trial`. - Simplest example: - ```python - def run_trial(self, trial, x, y, val_x, val_y): - model = self.hypermodel.build(trial.hyperparameters) - model.fit(x, y) - loss = model.evaluate(val_x, val_y) - self.oracle.update_trial( - trial.trial_id, {'loss': loss}) - self.save_model(trial.trial_id, model) - ``` - # Arguments: - trial: A `Trial` instance that contains the information - needed to run this trial. Hyperparameters can be accessed - via `trial.hyperparameters`. - *fit_args: Positional arguments passed by `search`. - *fit_kwargs: Keyword arguments passed by `search`. - """ - raise NotImplementedError - - def save_model(self, trial_id, model, step=0): - """Saves a Model for a given trial. - # Arguments: - trial_id: The ID of the `Trial` that corresponds to this Model. - model: The trained model. - step: For models that report intermediate results to the `Oracle`, - the step that this saved file should correspond to. For example, - for Keras models this is the number of epochs trained. - """ - raise NotImplementedError - - def load_model(self, trial): - """Loads a Model from a given trial. - # Arguments: - trial: A `Trial` instance. For models that report intermediate - results to the `Oracle`, generally `load_model` should load the - best reported `step` by relying of `trial.best_step` - """ - raise NotImplementedError - - def on_search_begin(self): - """A hook called at the beginning of `search`.""" - if self.logger: - self.logger.register_tuner(self.get_state()) - - def on_trial_begin(self, trial): - """A hook called before starting each trial. - # Arguments: - trial: A `Trial` instance. - """ - if self.logger: - self.logger.register_trial(trial.trial_id, trial.get_state()) - - def run_trial(self, trial, *fit_args, **fit_kwargs): - raise NotImplementedError - - def on_trial_end(self, trial, model): - """A hook called after each trial is run. - # Arguments: - trial: A `Trial` instance. - """ - # Send status to Logger - if self.logger: - self.logger.report_trial_state(trial.trial_id, trial.get_state()) - - self.oracle.end_trial( - trial.trial_id, trial_module.TrialStatus.COMPLETED) - self.save_weights(trial, model) - self.oracle.update_space(trial.hyperparameters) - self._display.on_trial_end(trial) - self.save() - - def on_search_end(self): - """A hook called at the end of `search`.""" - if self.logger: - self.logger.exit() - - def get_best_models(self, num_models=1): - """Returns the best model(s), as determined by the objective. - This method is only a convenience shortcut. For best performance, It is - recommended to retrain your Model on the full dataset using the best - hyperparameters found during `search`. - # Arguments: - num_models (int, optional). Number of best models to return. - Models will be returned in sorted order. Defaults to 1. - # Returns: - List of trained model instances. - """ - best_trials = self.oracle.get_best_trials(num_models) - models = [self.load_model(trial) for trial in best_trials] - return models - - def get_best_hyperparameters(self, num_trials=1): - """Returns the best hyperparameters, as determined by the objective. - This method can be used to reinstantiate the (untrained) best model - found during the search process. - Example: - ```python - best_hp = tuner.get_best_hyperparameters()[0] - model = tuner.hypermodel.build(best_hp) - ``` - # Arguments: - num_trials: (int, optional). Number of `HyperParameters` objects to - return. `HyperParameters` will be returned in sorted order based on - trial performance. - # Returns: - List of `HyperParameter` objects. - """ - return [t.hyperparameters for t in self.oracle.get_best_trials(num_trials)] - - def search_space_summary(self, extended=False): - """Print search space summary. - Args: - extended: Bool, optional. Display extended summary. - Defaults to False. - """ - display.section('Search space summary') - hp = self.oracle.get_space() - display.display_setting( - 'Default search space size: %d' % len(hp.space)) - for p in hp.space: - config = p.get_config() - name = config.pop('name') - if p.__class__.__name__ == 'Fixed': - continue - display.subsection('%s (%s)' % (name, p.__class__.__name__)) - display.display_settings(config) - - def results_summary(self, num_trials=10): - """Display tuning results summary. - Args: - num_trials (int, optional): Number of trials to display. - Defaults to 10. - sort_metric (str, optional): Sorting metric, when not specified - sort models by objective value. Defaults to None. - """ - display.section('Results summary') - display.display_setting('Results in %s' % self.project_dir) - best_trials = self.oracle.get_best_trials(num_trials) - display.display_setting('Showing %d best trials' % num_trials) - for trial in best_trials: - display.display_setting( - 'Objective: {} Score: {}'.format( - self.oracle.objective, trial.score)) - - @property - def remaining_trials(self): - """Returns the number of trials remaining. - Will return `None` if `max_trials` is not set. - """ - return self.oracle.remaining_trials() - - def get_state(self): - return {} - - def set_state(self, state): - pass - - def save(self): - super(BaseTuner, self).save(self._get_tuner_fname()) - - def reload(self): - super(BaseTuner, self).reload(self._get_tuner_fname()) - - @property - def project_dir(self): - dirname = os.path.join( - self.directory, - self.project_name) - create_directory(dirname) - return dirname - - def get_trial_dir(self, trial_id): - dirname = os.path.join( - self.project_dir, - 'trial_' + str(trial_id)) - create_directory(dirname) - return dirname - - def _get_tuner_fname(self): - return os.path.join( - self.project_dir, - str(self.tuner_id) + '.json') - - def get_best_models(self, num_models=1): - """Returns the best model(s), as determined by the tuner's objective. - - This method is only a convenience shortcut. - - Args: - num_models (int, optional): Number of best models to return. - Models will be returned in sorted order. Defaults to 1. - - Returns: - List of trained model instances. - """ - best_trials = self.oracle.get_best_trials(num_models) - models = [self.load_model(trial) for trial in best_trials] - return models - - def save_weights(self, trial, model): - raise NotImplementedError - - def load_model(self, trial): - raise NotImplementedError - - -class Tuner(BaseTuner): - """Tuner class for Keras models. - May be subclassed to create new tuners. - # Arguments: - oracle: Instance of Oracle class. - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - max_model_size: Int. Maximum size of weights - (in floating point coefficients) for a valid - models. Models larger than this are rejected. - optimizer: Optional. Optimizer instance. - May be used to override the `optimizer` - argument in the `compile` step for the - models. If the hypermodel - does not compile the models it generates, - then this argument must be specified. - loss: Optional. May be used to override the `loss` - argument in the `compile` step for the - models. If the hypermodel - does not compile the models it generates, - then this argument must be specified. - metrics: Optional. May be used to override the - `metrics` argument in the `compile` step - for the models. If the hypermodel - does not compile the models it generates, - then this argument must be specified. - directory: String. Path to the working directory (relative). - project_name: Name to use as prefix for files saved - by this Tuner. - logger: Optional. Instance of Logger class, used for streaming data - to Cloud Service for monitoring. - overwrite: Bool, default `False`. If `False`, reloads an existing project - of the same name if one is found. Otherwise, overwrites the project. - """ - - def __init__(self, - oracle, - max_model_size=None, - optimizer=None, - loss=None, - metrics=None, - directory=None, - project_name=None, - logger=None, - tuner_id=None, - overwrite=False): - - # Subclasses of `KerasHyperModel` are not automatically wrapped. - super(Tuner, self).__init__(oracle=oracle, - directory=directory, - project_name=project_name, - logger=logger, - overwrite=overwrite) - - # Save only the last N checkpoints. - self._save_n_checkpoints = 10 - - def run_trial(self, trial, *fit_args, **fit_kwargs): - """Evaluates a set of hyperparameter values. - This method is called during `search` to evaluate a set of - hyperparameters. - # Arguments: - trial: A `Trial` instance that contains the information - needed to run this trial. `Hyperparameters` can be accessed - via `trial.hyperparameters`. - *fit_args: Positional arguments passed by `search`. - *fit_kwargs: Keyword arguments passed by `search`. - """ - # Handle any callbacks passed to `fit`. - fit_kwargs = copy.copy(fit_kwargs) - callbacks = fit_kwargs.pop('callbacks', []) - callbacks = self._deepcopy_callbacks(callbacks) - self._configure_tensorboard_dir(callbacks, trial.trial_id) - # `TunerCallback` calls: - # - `Tuner.on_epoch_begin` - # - `Tuner.on_batch_begin` - # - `Tuner.on_batch_end` - # - `Tuner.on_epoch_end` - # These methods report results to the `Oracle` and save the trained Model. If - # you are subclassing `Tuner` to write a custom training loop, you should - # make calls to these methods within `run_trial`. - callbacks.append(TunerCallback(self, trial)) - - model = self.hypermodel.build(trial.hyperparameters) - model.fit(*fit_args, **fit_kwargs, callbacks=callbacks) - return model - - def save_model(self, trial_id, model, step=0): - epoch = step - self._checkpoint_model(model, trial_id, epoch) - if epoch > self._save_n_checkpoints: - self._delete_checkpoint( - trial_id, epoch - self._save_n_checkpoints) - - def load_model(self, trial): - model = self.hypermodel.build(trial.hyperparameters) - # Reload best checkpoint. The Oracle scores the Trial and also - # indicates at what epoch the best value of the objective was - # obtained. - best_epoch = trial.best_step - model.load_weights(self._get_checkpoint_fname( - trial.trial_id, best_epoch)) - return model - - def on_epoch_begin(self, trial, model, epoch, logs=None): - """A hook called at the start of every epoch. - # Arguments: - trial: A `Trial` instance. - model: A Keras `Model`. - epoch: The current epoch number. - logs: Additional metrics. - """ - pass - - def on_batch_begin(self, trial, model, batch, logs): - """A hook called at the start of every batch. - # Arguments: - trial: A `Trial` instance. - model: A Keras `Model`. - batch: The current batch number within the - curent epoch. - logs: Additional metrics. - """ - pass - - def on_batch_end(self, trial, model, batch, logs=None): - """A hook called at the end of every batch. - # Arguments: - trial: A `Trial` instance. - model: A Keras `Model`. - batch: The current batch number within the - curent epoch. - logs: Additional metrics. - """ - pass - - def on_epoch_end(self, trial, model, epoch, logs=None): - """A hook called at the end of every epoch. - # Arguments: - trial: A `Trial` instance. - model: A Keras `Model`. - epoch: The current epoch number. - logs: Dict. Metrics for this epoch. This should include - the value of the objective for this epoch. - """ - self.save_model(trial.trial_id, model, step=epoch) - # Report intermediate metrics to the `Oracle`. - status = self.oracle.update_trial( - trial.trial_id, metrics=logs, step=epoch) - trial.status = status - if trial.status == "STOPPED": - model.stop_training = True - - def get_best_models(self, num_models=1): - """Returns the best model(s), as determined by the tuner's objective. - The models are loaded with the weights corresponding to - their best checkpoint (at the end of the best epoch of best trial). - This method is only a convenience shortcut. For best performance, It is - recommended to retrain your Model on the full dataset using the best - hyperparameters found during `search`. - Args: - num_models (int, optional): Number of best models to return. - Models will be returned in sorted order. Defaults to 1. - Returns: - List of trained model instances. - """ - # Method only exists in this class for the docstring override. - return super(Tuner, self).get_best_models(num_models) - - def _deepcopy_callbacks(self, callbacks): - try: - callbacks = copy.deepcopy(callbacks) - except: - raise ValueError( - 'All callbacks used during a search ' - 'should be deep-copyable (since they are ' - 'reused across trials). ' - 'It is not possible to do `copy.deepcopy(%s)`' % - (callbacks,)) - return callbacks - - def _configure_tensorboard_dir(self, callbacks, trial_id): - for callback in callbacks: - # Patching tensorboard log dir - if callback.__class__.__name__ == 'TensorBoard': - callback.log_dir = os.path.join( - callback.log_dir, - str(trial_id)) - - def _get_checkpoint_dir(self, trial_id, epoch): - return os.path.join( - self.get_trial_dir(trial_id), - 'checkpoints', - 'epoch_' + str(epoch)) - - def _get_checkpoint_fname(self, trial_id, epoch): - return os.path.join( - # Each checkpoint is saved in its own directory. - self._get_checkpoint_dir(trial_id, epoch), - 'checkpoint') - - def _checkpoint_model(self, model, trial_id, epoch): - fname = self._get_checkpoint_fname(trial_id, epoch) - # Save in TF format. - model.save_weights(fname) - return fname - - def _delete_checkpoint(self, trial_id, epoch): - tf.io.gfile.rmtree(self._get_checkpoint_dir(trial_id, epoch)) - - -class MultiExecutionTuner(Tuner): - """A Tuner class that averages multiple runs of the process. - Args: - oracle: Instance of Oracle class. - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - executions_per_trial: Int. Number of executions - (training a model from scratch, - starting from a new initialization) - to run per trial (model configuration). - Model metrics may vary greatly depending - on random initialization, hence it is - often a good idea to run several executions - per trial in order to evaluate the performance - of a given set of hyperparameter values. - **kwargs: Keyword arguments relevant to all `Tuner` subclasses. - Please see the docstring for `Tuner`. - """ - - def __init__(self, - oracle, - executions_per_trial=1, - **kwargs): - super(MultiExecutionTuner, self).__init__( - oracle, **kwargs) - if isinstance(oracle.objective, list): - raise ValueError( - 'Multi-objective is not supported, found: {}'.format( - oracle.objective)) - self.executions_per_trial = executions_per_trial - # This is the `step` that will be reported to the Oracle at the end - # of the Trial. Since intermediate results are not used, this is set - # to 0. - self._reported_step = 0 - - def on_epoch_end(self, trial, model, epoch, logs=None): - # Intermediate results are not passed to the Oracle, and - # checkpointing is handled via a `ModelCheckpoint` callback. - pass - - def run_trial(self, trial, *fit_args, **fit_kwargs): - model_checkpoint = tf.keras.callbacks.ModelCheckpoint( - filepath=self._get_checkpoint_fname( - trial.trial_id, self._reported_step), - monitor=self.oracle.objective.name, - mode=self.oracle.objective.direction, - save_best_only=True, - save_weights_only=True) - - # Run the training process multiple times. - metrics = collections.defaultdict(list) - for execution in range(self.executions_per_trial): - fit_kwargs = copy.copy(fit_kwargs) - callbacks = fit_kwargs.pop('callbacks', []) - callbacks = self._deepcopy_callbacks(callbacks) - self._configure_tensorboard_dir(callbacks, trial.trial_id, execution) - callbacks.append(TunerCallback(self, trial)) - # Only checkpoint the best epoch across all executions. - callbacks.append(model_checkpoint) - - model = self.hypermodel.build(trial.hyperparameters) - # model.summary() - history = model.fit(*fit_args, **fit_kwargs, callbacks=callbacks) - - for metric, epoch_values in history.history.items(): - if self.oracle.objective.direction == 'min': - best_value = np.min(epoch_values) - else: - best_value = np.max(epoch_values) - metrics[metric].append(best_value) - - # Average the results across executions and send to the Oracle. - averaged_metrics = {} - for metric, execution_values in metrics.items(): - averaged_metrics[metric] = np.mean(execution_values) - self.oracle.update_trial( - trial.trial_id, metrics=averaged_metrics, step=self._reported_step) - return model - - def _configure_tensorboard_dir(self, callbacks, trial_id, execution=0): - for callback in callbacks: - # Patching tensorboard log dir - if callback.__class__.__name__ == 'TensorBoard': - callback.log_dir = os.path.join( - callback.log_dir, - trial_id, - 'execution{}'.format(execution)) - return callbacks - - -class PipeTuner(MultiExecutionTuner): - - def __init__(self, oracle, hypergraph, fit_on_val_data=False, **kwargs): - super().__init__(oracle, **kwargs) - self.oracle = oracle - self.hypergraph = hypergraph - self.need_fully_train = False - self.best_hp = None - self.fit_on_val_data = fit_on_val_data - - def run_trial(self, trial, *fit_args, **fit_kwargs): - """Preprocess the x and y before calling the base run_trial.""" - # Initialize new fit kwargs for the current trial. - fit_kwargs.update( - dict(zip(inspect.getfullargspec(tf.keras.Model.fit).args, fit_args))) - new_fit_kwargs = copy.copy(fit_kwargs) - - # Preprocess the dataset and set the shapes of the HyperNodes. - self.hypermodel = self.hypergraph.build_graphs(trial.hyperparameters) - - self._prepare_run(new_fit_kwargs) - - model = super().run_trial(trial, **new_fit_kwargs) - return model - - def _prepare_run(self, fit_kwargs): - validation_data = (fit_kwargs.pop('x_val', None), fit_kwargs.pop('y_val', None)) - - # Update the new fit kwargs values - fit_kwargs['x'] = fit_kwargs.get('x', None) - fit_kwargs['y'] = fit_kwargs.get('y', None) - fit_kwargs['validation_data'] = validation_data - fit_kwargs['batch_size'] = fit_kwargs.get('batch_size', 32) - - def save_weights(self, trial, pipe): - trial_dir = self.get_trial_dir(trial.trial_id) - tf.keras.models.save_model(pipe, trial_dir) - - def load_model(self, trial): - """Load the model in a history trial. - # Arguments - trial: Trial. The trial to be loaded. - # Returns - Tuple of (PreprocessGraph, KerasGraph, tf.keras.Model). - """ - models = tf.keras.models.load_model(self.get_trial_dir(trial.trial_id), compile=False) - models.compile(loss=tf.keras.losses.BinaryCrossentropy()) - self.hypermodel = None - return models - - def get_best_model(self): - """Load the best PreprocessGraph and Keras model. - It is mainly used by the predict and evaluate function of AutoModel. - # Returns - tf.keras.Model - """ - keras_graph = self.hypergraph.build_graphs( - self.best_hp) - keras_graph.reload(self.best_keras_graph_path) - model = keras_graph.build(self.best_hp) - model.load_weights(self.best_model_path) - return model - - @property - def best_keras_graph_path(self): - return os.path.join(self.project_dir, 'best_keras_graph') - - @property - def best_model_path(self): - return os.path.join(self.project_dir, 'best_model') - - def _get_save_path(self, trial, name): - filename = '{trial_id}-{name}'.format(trial_id=trial.trial_id, name=name) - return os.path.join(self.get_trial_dir(trial.trial_id), filename) - - def on_trial_end(self, trial, model): - """Save and clear the hypermodel.""" - super().on_trial_end(trial, model) - - self.hypermodel.save(self._get_save_path(trial, 'keras_graph')) - self.hypermodel = None - - diff --git a/examples/ctr_autoint.py b/examples/ctr_autoint.py index b82a0b4..0044086 100644 --- a/examples/ctr_autoint.py +++ b/examples/ctr_autoint.py @@ -2,29 +2,35 @@ from __future__ import absolute_import, division, print_function, unicode_literals import os + os.environ["CUDA_VISIBLE_DEVICES"] = "7" import logging import tensorflow as tf import numpy as np -from autorecsys.auto_search import Search -from autorecsys.pipeline import Input, DenseFeatureMapper, SparseFeatureMapper, SelfAttentionInteraction, MLPInteraction, PointWiseOptimizer -from autorecsys.recommender import CTRRecommender +# from autorecsys.auto_search import Search +from autorecsys.pipeline import DenseFeatureMapper, SparseFeatureMapper, SelfAttentionInteraction, MLPInteraction, \ + PointWiseOptimizer +# from autorecsys.pipeline import Input +# from autorecsys.recommender import CTRRecommender +import numpy as np +import autokeras as ak +import tensorflow as tf # logging setting logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) +# sync # load dataset -mini_criteo = np.load("./examples/datasets/criteo/criteo_2M.npz") +mini_criteo = np.load("datasets/criteo_ctr/criteo_1000.npz") # TODO: preprocess train val split train_X = [mini_criteo['X_int'].astype(np.float32), mini_criteo['X_cat'].astype(np.float32)] train_y = mini_criteo['y'] val_X, val_y = train_X, train_y - # build the pipeline. -dense_input_node = Input(shape=[13]) -sparse_input_node = Input(shape=[26]) +dense_input_node = ak.Input(shape=[13]) +sparse_input_node = ak.Input(shape=[26]) dense_feat_emb = DenseFeatureMapper( num_of_fields=13, embedding_dim=2)(dense_input_node) @@ -45,22 +51,30 @@ bottom_mlp_output = MLPInteraction()([dense_feat_emb]) top_mlp_output = MLPInteraction()([attention_output, bottom_mlp_output]) -output = PointWiseOptimizer()(top_mlp_output) -model = CTRRecommender(inputs=[dense_input_node, sparse_input_node], outputs=output) +# output = PointWiseOptimizer()(top_mlp_output) +output = ak.ClassificationHead()(top_mlp_output) +# model = CTRRecommender(inputs=[dense_input_node, sparse_input_node], outputs=output) + +auto_model = ak.AutoModel(inputs=[dense_input_node, sparse_input_node], + outputs=output, + overwrite=True, + max_trials=2) +auto_model.fit(train_X, train_y, epochs=1) +print(auto_model.evaluate(val_X, val_y)) # AutoML search and predict. -searcher = Search(model=model, - tuner='random', - tuner_params={'max_trials': 2, 'overwrite': True}, - ) -searcher.search(x=train_X, - y=train_y, - x_val=val_X, - y_val=val_y, - objective='val_BinaryCrossentropy', - batch_size=10000, - epochs = 20, - callbacks = [ tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=1)] - ) -logger.info('First 10 Predicted Ratings: {}'.format(searcher.predict(x=val_X)[:10])) -logger.info('Predicting Accuracy (logloss): {}'.format(searcher.evaluate(x=val_X, y_true=val_y))) +# searcher = Search(model=model, +# tuner='random', +# tuner_params={'max_trials': 2, 'overwrite': True}, +# ) +# searcher.search(x=train_X, +# y=train_y, +# x_val=val_X, +# y_val=val_y, +# objective='val_BinaryCrossentropy', +# batch_size=10000, +# epochs = 20, +# callbacks = [ tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=1)] +# ) +# logger.info('First 10 Predicted Ratings: {}'.format(searcher.predict(x=val_X)[:10])) +# logger.info('Predicting Accuracy (logloss): {}'.format(searcher.evaluate(x=val_X, y_true=val_y))) diff --git a/examples/rp_autorec.py b/examples/rp_autorec.py index d245394..3b353e9 100644 --- a/examples/rp_autorec.py +++ b/examples/rp_autorec.py @@ -1,16 +1,20 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function, unicode_literals +import sys + +#sys.path.append('/home/suil5044/AutoRec/sync/examples') import os + os.environ["CUDA_VISIBLE_DEVICES"] = "2" import logging import tensorflow as tf -from autorecsys.auto_search import Search -from autorecsys.pipeline import Input, LatentFactorMapper, RatingPredictionOptimizer, HyperInteraction -from autorecsys.pipeline.preprocessor import MovielensPreprocessor -from autorecsys.recommender import RPRecommender +import autokeras as ak +from autorecsys.pipeline import SparseFeatureMapper, DenseFeatureMapper, LatentFactorMapper, RatingPredictionOptimizer, HyperInteraction +from autorecsys.pipeline.preprocessor import MovielensPreprocessor, MovielensPreprocessor2 +#### # logging setting logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -21,8 +25,8 @@ # dataset_paths = ["./examples/datasets/netflix-prize-data/combined_data_" + str(i) + ".txt" for i in range(1, 5)] # data = NetflixPrizePreprocessor(dataset_paths) -#Movielens 1M Dataset -data = MovielensPreprocessor("./examples/datasets/ml-1m/ratings.dat") +# Movielens 1M Dataset +data = MovielensPreprocessor("datasets/movielens_rp/ml-1m/ratings.dat") ##Movielens 10M Dataset # data = MovielensPreprocessor("./examples/datasets/ml-10M100K/ratings.dat") @@ -32,6 +36,7 @@ data.preprocessing(val_test_size=0.1, random_state=1314) train_X, train_y = data.train_X, data.train_y + val_X, val_y = data.val_X, data.val_y test_X, test_y = data.test_X, data.test_y user_num, item_num = data.user_num, data.item_num @@ -43,34 +48,34 @@ logger.info('test_y size: {}'.format(test_y.shape)) logger.info('user total number: {}'.format(user_num)) logger.info('item total number: {}'.format(item_num)) - - +#### # build the pipeline. -input = Input(shape=[2]) +input = ak.Input(shape=[2]) +# sparse_input = SparseFeatureMapper( +# num_of_fields=2, +# embedding_dim=64)(input) +# dense_feat_emb = DenseFeatureMapper( +# num_of_fields=2, +# embedding_dim=2)(input) + user_emb = LatentFactorMapper(feat_column_id=0, id_num=user_num, embedding_dim=64)(input) item_emb = LatentFactorMapper(feat_column_id=1, id_num=item_num, embedding_dim=64)(input) -output1 = HyperInteraction()([user_emb, item_emb]) + +output1 = HyperInteraction()([input]) output2 = HyperInteraction()([output1, user_emb, item_emb]) output3 = HyperInteraction()([output1, output2, user_emb, item_emb]) output4 = HyperInteraction()([output1, output2, output3, user_emb, item_emb]) -output = RatingPredictionOptimizer()(output4) -model = RPRecommender(inputs=input, outputs=output) -# AutoML search and predict. -searcher = Search(model=model, - tuner='random', ## hyperband, bayesian - tuner_params={'max_trials': 100, 'overwrite': True},) -searcher.search(x=train_X, - y=train_y, - x_val=val_X, - y_val=val_y, - objective='val_mse', - batch_size=1024, - epochs=10, - callbacks=[tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=1)]) -logger.info('Predicting Val Dataset Accuracy (mse): {}'.format(searcher.evaluate(x=val_X, y_true=val_y))) -logger.info('Predicting Test Dataset Accuracy (mse): {}'.format(searcher.evaluate(x=test_X, y_true=test_y))) +output = ak.ClassificationHead()(output1) +# model = CTRRecommender(inputs=[dense_input_node, sparse_input_node], outputs=output) + +auto_model = ak.AutoModel(inputs=input, + outputs=output, + overwrite=True, + max_trials=2) +auto_model.fit(train_X, train_y, epochs=2) +print(auto_model.evaluate(val_X, val_y)) \ No newline at end of file