# Copyright 2016 James Hensman, Mark van der Wilk, # Valentine Svensson, alexggmatthews, # PabloLeon, fujiisoup # Copyright 2017 Artem Artemev @awav # # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import enum import numpy as np import tensorflow as tf from .. import settings from ..core.errors import GPflowError from ..core.compilable import Build from ..core.node import Node from ..core.base import IPrior, ITransform from .. import misc from ..transforms import Identity EMPTY_FEEDS = {} class Parameter(Node): """ Parameter class is a cornerstone of the GPflow package. It wraps TensorFlow variable and its prior and transformation building operations. In GPflow computation graph the parameter is a leaf in the tree. *Constrained* and *unconstrained* values of the parameter. These definitions arise from optimization topic, constrained and unconstrained optimization accordingly. _Constrained_ value has acceptable value boundaries and user always works with such values. It means that when you pass value to create parameter object, the parameter assumes that this is _constrained_ value. When user reads value of the parameter, the user gets _constrained_ value. For example, variance cannot be negative, hence the constraint is [0, ∞). ``` param = gpflow.Param(1.0) # Here `1.0` is constrained value. value = param.read_value() # `value` is constrained value too. ``` *Unconstrained* value doesn't have any limits. The necessity in this value is caused by available set of TensorFlow optimizers. In fact, internal parameter tensor is unconstrained and will be trained as unconstrained variable. ``` param = gpflow.Param(1.0) # `1.0` is constrained value. param.parameter_tensor # unconstrained TensorFlow variable. ``` User can manage contraint type which will be used at a parameter. There is special *transform* property and option in constructor for that. The user can pass one of the implementations of `ITransform` interface. By default Identity transform is applied, so that constrained has no difference from unconstrained, and variable is simply considered as unconstrained. ``` param = gpflow.Param(1.0) # Identity transform is applied. ``` In example below, the parameter has exponential transform. It means that input value is always must be positive [0, ∞), but internal unconstrained tensor has no boundaries (∞, ∞). ``` param = gpflow.Param(1.0, transform=gpflow.transforms.Exp()) param.read_value() # 1.0 gpflow.get_default_session().run(param.parameter_tensor) # -1.0000005000290891e-06 ``` *Parameter's shape*. The parameter's shape is always fixed by default. It means that user is not allowed to modify shape when parameter's tensors are built. User can modify default behavior by setting up `fix_shape` option to `False`, the parameter will be able to change its shape and internal parameter's tensor will have floating shape - None. *NOTE: trainable parameters with floating shape cannot be trained by a bunch of TensorFlow optimizers like RMSProp, Adam as they require shape information for trainable variables in advance of optimizer's tensors construction.* :param value: Constrained input value. It can be a float, an integer, a float or integer like list, numpy array or TensorFlow variable. :param transform: Instance of GPflow.ITransform implementation. :param prior: Instance of GPflow.IPrior implementation. :param trainable: Boolean flag. It indicates whether variables will be added to trainbles TensorFlow set or not. :param dtype: Type of new parameter. :param fix_shape: Default value is `True` and indicates that shape of internal tensor will be the same as the shape of the input variable and can not be changed. `False` will turn on floating shape mode for the tensor. :param name: Name of the parameter. :raises: ValueError exception if value is not valid. In cases when default graph used for building parameter differs from the graph used during construction of priors or transformations, the GPflowError exception is raised. """ class ParameterAttribute(enum.Enum): PRIOR = 'prior' TRANSFORM = 'transform' TRAINABLE = 'trainable' @property def interface(self): if self.value == self.PRIOR.value: return IPrior elif self.value == self.TRANSFORM.value: return ITransform def __init__(self, value, transform=None, prior=None, trainable=True, dtype=None, fix_shape=True, name=None): self._externally_defined = False self._fixed_shape = fix_shape value = self._valid_input(value, dtype=dtype) super().__init__(name) self._init_parameter_defaults() self._init_parameter_attributes(prior, transform, trainable) self._init_parameter_value(value) @property def shape(self): if self._externally_defined: return tuple(self.parameter_tensor.shape.as_list()) return self._value.shape @property def dtype(self): if self._externally_defined: return self.parameter_tensor.dtype.as_numpy_dtype return self._value.dtype @property def size(self): """The size of this parameter, equivalent to self.value.size""" return np.multiply.reduce(self.shape, dtype=np.int32) @property def parameter_tensor(self): return self._unconstrained_tensor @property def unconstrained_tensor(self): return self._unconstrained_tensor @property def constrained_tensor(self): return self._constrained_tensor @property def prior_tensor(self): return self._prior_tensor @property def feeds(self): return EMPTY_FEEDS @property def initializables(self): if self._externally_defined: return None return [(self.parameter_tensor, self.is_initialized_tensor)] @property def initializable_feeds(self): if self._externally_defined: return EMPTY_FEEDS return {self._initial_value_tensor: self._apply_transform(self._value)} @property def graph(self): if self.parameter_tensor is None: return None return self.parameter_tensor.graph @property def fixed_shape(self): return self._fixed_shape @property def value(self): """ The `value` property is simple alias for `read_value()` method called with default arguments. """ return self.read_value() @property def is_initialized_tensor(self): """ :returns: Initialization status boolean tensor for parameter's variable. """ return self._is_initialized_tensor def is_initialized(self, session): if not isinstance(session, tf.Session): raise ValueError('TensorFlow session expected for initializing test.') if self.parameter_tensor is None: return False return session.run(self.is_initialized_tensor) def fix_shape(self): if self._fixed_shape: return if self.parameter_tensor is not None: self.parameter_tensor.set_shape(self.shape) self._fixed_shape = True def anchor(self, session): if not isinstance(session, tf.Session): ValueError('TensorFlow Session expected when anchoring.') if self.trainable: value = self.read_value(session=session) self.assign(value, session=session) def is_built(self, graph): if not isinstance(graph, tf.Graph): raise ValueError('TensorFlow graph expected for build status testing.') if self.graph and self.graph is not graph: return Build.NOT_COMPATIBLE_GRAPH elif self.prior_tensor is None: return Build.NO return Build.YES @property def trainable(self): return self._trainable @trainable.setter def trainable(self, value): self.set_trainable(value) def set_trainable(self, value): if not isinstance(value, bool): raise ValueError('Fixed property value must be boolean.') if self._externally_defined: raise GPflowError('Externally defined parameter tensor is not modifiable.') graph = self.graph if graph is not None: if self.trainable == value: return if value: misc.add_to_trainables(self.parameter_tensor, graph) else: misc.remove_from_trainables(self.parameter_tensor, graph) self._trainable = value def assign(self, value, session=None, dtype=None, force=True): if self._externally_defined: raise GPflowError("Externally defined parameter tensor is not modifiable.") value = self._valid_input(value, dtype) if self.fixed_shape: self._value[...] = value else: self._value = value.copy() if self.is_built_coherence() is Build.YES: session = self.enquire_session(session) self.initialize(session=session, force=force) def read_value(self, session=None): if session is not None and not isinstance(session, tf.Session): raise ValueError('TensorFlow session expected as an argument.') if session is None and self._externally_defined: raise GPflowError('Externally defined parameter requires session.') elif session: is_built = self.is_built_coherence(session.graph) if is_built is Build.YES: return self._read_parameter_tensor(session) return self._value def as_pandas_table(self): column_names = ['class', 'prior', 'transform', 'trainable', 'shape', 'fixed_shape', 'value'] column_values = [self.__class__.__name__, str(self.prior), str(self.transform), self.trainable, self.shape, self.fixed_shape, self.value.copy()] column_values = [[value] for value in column_values] df = misc.pretty_pandas_table([self.pathname], column_names, column_values) return df def tf_compilation_index(self): """ Takes out index from the parameter's tensor name. E.g. parameter tensor name is GPR-0000/kern/lengthscales, the method for that parameter will return '0000' index. """ if self.parameter_tensor is None: return None name = self.parameter_tensor.name return name.split('-', 1)[-1].split('/')[0] def _valid_input(self, value, dtype=None): if not misc.is_valid_param_value(value): msg = 'The value must be either a tensorflow variable, an array or a scalar.' raise ValueError(msg) cast = not (dtype is None) is_built = False shape = None if hasattr(self, '_value'): # The parameter has not initialized yet. is_built = self.is_built_coherence() == Build.YES shape = self.shape inner_dtype = self.dtype if dtype is not None and inner_dtype != dtype: msg = 'Overriding parameter\'s type "{0}" with "{1}" is not possible.' raise ValueError(msg.format(inner_dtype, dtype)) elif isinstance(value, np.ndarray) and inner_dtype != value.dtype: msg = 'The value has different data type "{0}". Parameter type is "{1}".' raise ValueError(msg.format(value.dtype, inner_dtype)) cast = False dtype = inner_dtype if misc.is_number(value): value_type = np.result_type(value).type num_type = misc.normalize_num_type(value_type) dtype = num_type if dtype is None else dtype value = np.array(value, dtype=dtype) elif misc.is_list(value): dtype = settings.float_type if dtype is None else dtype value = np.array(value, dtype=dtype) elif cast and not misc.is_tensor(value): value = value.astype(dtype) if shape is not None and self.fixed_shape and is_built and shape != value.shape: msg = 'Value has different shape. Parameter shape {0}, value shape {1}.' raise ValueError(msg.format(shape, value.shape)) return value def _clear(self): self.reset_name() self._externally_defined = False self._is_initialized_tensor = None self._initial_value_tensor = None self._unconstrained_tensor = None self._constrained_tensor = None self._prior_tensor = None def _build(self): unconstrained = self._build_parameter() constrained = self._build_constrained(unconstrained) prior = self._build_prior(unconstrained, constrained) self._is_initialized_tensor = True if not isinstance(unconstrained, tf.Tensor): self._is_initialized_tensor = tf.is_variable_initialized(unconstrained) self._unconstrained_tensor = unconstrained self._constrained_tensor = constrained self._prior_tensor = prior def _build_parameter(self): if self._externally_defined: self._check_tensor_trainable(self.parameter_tensor) return self.parameter_tensor name = self._parameter_name() tensor = misc.get_variable_by_name(name) if tensor is not None: raise GPflowError('Tensor with name "{name}" already exists, {tensor}.' .format(name=name, tensor=tensor)) value = self._apply_transform(self._value) shape = value.shape if self.fixed_shape else None init = tf.placeholder(self.dtype, shape=shape, name='initial_unconstrained_value') self._initial_value_tensor = init if self.fixed_shape: args = dict(trainable=self.trainable) else: args = dict(validate_shape=False, trainable=self.trainable) variable = tf.get_variable(name, initializer=init, **args) return variable def _build_constrained(self, parameter_tensor): if not misc.is_tensor(parameter_tensor): # pragma: no cover raise GPflowError("Input must be a tensor.") return self.transform.forward_tensor(parameter_tensor) def _build_prior(self, unconstrained_tensor, constrained_tensor): """ Build a tensorflow representation of the prior density. The log Jacobian is included. """ if not misc.is_tensor(unconstrained_tensor): raise GPflowError("Unconstrained input must be a tensor.") if not misc.is_tensor(constrained_tensor): raise GPflowError("Constrained input must be a tensor.") prior_name = 'prior' if self.prior is None: return tf.constant(0.0, settings.float_type, name=prior_name) log_jacobian = self.transform.log_jacobian_tensor(unconstrained_tensor) logp_var = self.prior.logp(constrained_tensor) return tf.squeeze(tf.add(logp_var, log_jacobian, name=prior_name)) def _check_tensor_trainable(self, tensor): is_trainable = misc.is_tensor_trainable(tensor) if is_trainable != self.trainable: tensor_status = 'trainable' if is_trainable else 'not trainable' param_status = 'trainable' if self.trainable else 'not' msg = 'Externally defined tensor is {0} whilst parameter is {1}.' raise GPflowError(msg.format(tensor_status, param_status)) def _init_parameter_defaults(self): self._is_initialized_tensor = None self._initial_value_tensor = None self._unconstrained_tensor = None self._prior_tensor = None self._constrained_tensor = None def _init_parameter_value(self, value): if misc.is_tensor(value): is_trainable = misc.is_tensor_trainable(value) if is_trainable != self.trainable: status = 'trainable' if is_trainable else 'not trainable' raise ValueError('Externally defined tensor is {0}.'.format(status)) self._fixed_shape = True self._externally_defined = True self._set_parameter_tensor(value) else: self._value = value.copy() def _init_parameter_attributes(self, prior, transform, trainable): if transform is None: transform = Identity() self.prior = prior # pylint: disable=W0201 self.transform = transform # pylint: disable=W0201 self.trainable = trainable # pylint: disable=W0201 def _read_parameter_tensor(self, session): return session.run(self.constrained_tensor) def _apply_transform(self, value): return self.transform.backward(value) def _parameter_name(self): return misc.tensor_name(self.tf_pathname, 'unconstrained') def _set_parameter_tensor(self, tensor): self._unconstrained_tensor = tensor def _set_parameter_attribute(self, attr, value): if attr is self.ParameterAttribute.TRAINABLE: self.set_trainable(value) return is_built = self.is_built_coherence(self.graph) if is_built is Build.YES: raise GPflowError('Parameter "{}" has already been compiled.'.format(self.pathname)) name = attr.value if value is not None and not isinstance(value, attr.interface): msg = 'Attribute "{0}" must implement interface "{1}".' raise GPflowError(msg.format(name, attr.interface)) object.__setattr__(self, name, value) def __setattr__(self, name, value): try: attr = self.ParameterAttribute[name.upper()] self._set_parameter_attribute(attr, value) return except KeyError: pass object.__setattr__(self, name, value) def __str__(self): return str(self.as_pandas_table()) def _repr_html_(self): return self.as_pandas_table()._repr_html_() @property def fixed(self): raise NotImplementedError("`fixed` property is no longer supported. Please use `trainable` instead.") @fixed.setter def fixed(self, _): raise NotImplementedError("`fixed` property is no longer supported. Please use `trainable` instead.")