"""Python wrappers around TensorFlow ops. This file is MACHINE GENERATED! Do not edit. Original C++ source file: training_ops.cc """ import collections from tensorflow.python import pywrap_tfe as pywrap_tfe from tensorflow.python.eager import context as _context from tensorflow.python.eager import core as _core from tensorflow.python.eager import execute as _execute from tensorflow.python.framework import dtypes as _dtypes from tensorflow.python.framework import op_def_registry as _op_def_registry from tensorflow.python.framework import ops as _ops from tensorflow.python.framework import op_def_library as _op_def_library from tensorflow.python.util.deprecation import deprecated_endpoints from tensorflow.python.util import dispatch as _dispatch from tensorflow.python.util.tf_export import tf_export from typing import TypeVar def apply_ada_max(var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the AdaMax algorithm. m_t <- beta1 * m_{t-1} + (1 - beta1) * g v_t <- max(beta2 * v_{t-1}, abs(g)) variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon) Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). m: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). v: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). beta1_power: A `Tensor`. Must have the same type as `var`. Must be a scalar. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. beta1: A `Tensor`. Must have the same type as `var`. Momentum factor. Must be a scalar. beta2: A `Tensor`. Must have the same type as `var`. Momentum factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, m, and v tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_ada_max op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAdaMax", var=var, m=m, v=v, beta1_power=beta1_power, lr=lr, beta1=beta1, beta2=beta2, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAdaMax", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAdaMax = tf_export("raw_ops.ApplyAdaMax")(_ops.to_raw_op(apply_ada_max)) def apply_ada_max_eager_fallback(var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad, use_locking, name, ctx): raise RuntimeError("apply_ada_max op does not support eager execution. Arg 'out' is a ref.") def apply_adadelta(var, accum, accum_update, lr, rho, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the adadelta scheme. accum = rho() * accum + (1 - rho()) * grad.square(); update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad; update_accum = rho() * update_accum + (1 - rho()) * update.square(); var -= update; Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). accum_update: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `var`. Decay factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var, accum and update_accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_adadelta op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAdadelta", var=var, accum=accum, accum_update=accum_update, lr=lr, rho=rho, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAdadelta", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAdadelta = tf_export("raw_ops.ApplyAdadelta")(_ops.to_raw_op(apply_adadelta)) def apply_adadelta_eager_fallback(var, accum, accum_update, lr, rho, epsilon, grad, use_locking, name, ctx): raise RuntimeError("apply_adadelta op does not support eager execution. Arg 'out' is a ref.") def apply_adagrad(var, accum, lr, grad, use_locking=False, update_slots=True, name=None): r"""Update '*var' according to the adagrad scheme. accum += grad * grad var -= lr * grad * (1 / sqrt(accum)) Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_adagrad op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAdagrad", var=var, accum=accum, lr=lr, grad=grad, use_locking=use_locking, update_slots=update_slots, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking"), "update_slots", _op._get_attr_bool("update_slots")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAdagrad", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAdagrad = tf_export("raw_ops.ApplyAdagrad")(_ops.to_raw_op(apply_adagrad)) def apply_adagrad_eager_fallback(var, accum, lr, grad, use_locking, update_slots, name, ctx): raise RuntimeError("apply_adagrad op does not support eager execution. Arg 'out' is a ref.") def apply_adagrad_da(var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step, use_locking=False, name=None): r"""Update '*var' according to the proximal adagrad scheme. Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). gradient_accumulator: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). gradient_squared_accumulator: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). grad: A `Tensor`. Must have the same type as `var`. The gradient. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. global_step: A `Tensor` of type `int64`. Training step number. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_adagrad_da op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAdagradDA", var=var, gradient_accumulator=gradient_accumulator, gradient_squared_accumulator=gradient_squared_accumulator, grad=grad, lr=lr, l1=l1, l2=l2, global_step=global_step, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAdagradDA", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAdagradDA = tf_export("raw_ops.ApplyAdagradDA")(_ops.to_raw_op(apply_adagrad_da)) def apply_adagrad_da_eager_fallback(var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step, use_locking, name, ctx): raise RuntimeError("apply_adagrad_da op does not support eager execution. Arg 'out' is a ref.") def apply_adagrad_v2(var, accum, lr, epsilon, grad, use_locking=False, update_slots=True, name=None): r"""Update '*var' according to the adagrad scheme. accum += grad * grad var -= lr * grad * (1 / sqrt(accum)) Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_adagrad_v2 op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAdagradV2", var=var, accum=accum, lr=lr, epsilon=epsilon, grad=grad, use_locking=use_locking, update_slots=update_slots, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking"), "update_slots", _op._get_attr_bool("update_slots")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAdagradV2", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAdagradV2 = tf_export("raw_ops.ApplyAdagradV2")(_ops.to_raw_op(apply_adagrad_v2)) def apply_adagrad_v2_eager_fallback(var, accum, lr, epsilon, grad, use_locking, update_slots, name, ctx): raise RuntimeError("apply_adagrad_v2 op does not support eager execution. Arg 'out' is a ref.") def apply_adam(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking=False, use_nesterov=False, name=None): r"""Update '*var' according to the Adam algorithm. $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$ $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$ $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$ $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). m: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). v: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). beta1_power: A `Tensor`. Must have the same type as `var`. Must be a scalar. beta2_power: A `Tensor`. Must have the same type as `var`. Must be a scalar. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. beta1: A `Tensor`. Must have the same type as `var`. Momentum factor. Must be a scalar. beta2: A `Tensor`. Must have the same type as `var`. Momentum factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, m, and v tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, uses the nesterov update. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_adam op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAdam", var=var, m=m, v=v, beta1_power=beta1_power, beta2_power=beta2_power, lr=lr, beta1=beta1, beta2=beta2, epsilon=epsilon, grad=grad, use_locking=use_locking, use_nesterov=use_nesterov, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking"), "use_nesterov", _op._get_attr_bool("use_nesterov")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAdam", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAdam = tf_export("raw_ops.ApplyAdam")(_ops.to_raw_op(apply_adam)) def apply_adam_eager_fallback(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking, use_nesterov, name, ctx): raise RuntimeError("apply_adam op does not support eager execution. Arg 'out' is a ref.") def apply_add_sign(var, m, lr, alpha, sign_decay, beta, grad, use_locking=False, name=None): r"""Update '*var' according to the AddSign update. m_t <- beta1 * m_{t-1} + (1 - beta1) * g update <- (alpha + sign_decay * sign(g) *sign(m)) * g variable <- variable - lr_t * update Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). m: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. alpha: A `Tensor`. Must have the same type as `var`. Must be a scalar. sign_decay: A `Tensor`. Must have the same type as `var`. Must be a scalar. beta: A `Tensor`. Must have the same type as `var`. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and m tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_add_sign op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyAddSign", var=var, m=m, lr=lr, alpha=alpha, sign_decay=sign_decay, beta=beta, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyAddSign", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyAddSign = tf_export("raw_ops.ApplyAddSign")(_ops.to_raw_op(apply_add_sign)) def apply_add_sign_eager_fallback(var, m, lr, alpha, sign_decay, beta, grad, use_locking, name, ctx): raise RuntimeError("apply_add_sign op does not support eager execution. Arg 'out' is a ref.") def apply_centered_rms_prop(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the centered RMSProp algorithm. The centered RMSProp algorithm uses an estimate of the centered second moment (i.e., the variance) for normalization, as opposed to regular RMSProp, which uses the (uncentered) second moment. This often helps with training, but is slightly more expensive in terms of computation and memory. Note that in dense implementation of this algorithm, mg, ms, and mom will update even if the grad is zero, but in this sparse implementation, mg, ms, and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 mean_grad = decay * mean_grad + (1-decay) * gradient Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2) mg <- rho * mg_{t-1} + (1-rho) * grad ms <- rho * ms_{t-1} + (1-rho) * grad * grad mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon) var <- var - mom Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). mg: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). ms: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). mom: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `var`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `var`. Momentum Scale. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, mg, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_centered_rms_prop op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyCenteredRMSProp", var=var, mg=mg, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyCenteredRMSProp", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyCenteredRMSProp = tf_export("raw_ops.ApplyCenteredRMSProp")(_ops.to_raw_op(apply_centered_rms_prop)) def apply_centered_rms_prop_eager_fallback(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, use_locking, name, ctx): raise RuntimeError("apply_centered_rms_prop op does not support eager execution. Arg 'out' is a ref.") def apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update '*var' according to the Ftrl-proximal scheme. accum_new = accum + grad * grad linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). linear: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). grad: A `Tensor`. Must have the same type as `var`. The gradient. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. lr_power: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_ftrl op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyFtrl", var=var, accum=accum, linear=linear, grad=grad, lr=lr, l1=l1, l2=l2, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking"), "multiply_linear_by_lr", _op._get_attr_bool("multiply_linear_by_lr")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyFtrl", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyFtrl = tf_export("raw_ops.ApplyFtrl")(_ops.to_raw_op(apply_ftrl)) def apply_ftrl_eager_fallback(var, accum, linear, grad, lr, l1, l2, lr_power, use_locking, multiply_linear_by_lr, name, ctx): raise RuntimeError("apply_ftrl op does not support eager execution. Arg 'out' is a ref.") def apply_ftrl_v2(var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update '*var' according to the Ftrl-proximal scheme. grad_with_shrinkage = grad + 2 * l2_shrinkage * var accum_new = accum + grad * grad linear += grad_with_shrinkage - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). linear: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). grad: A `Tensor`. Must have the same type as `var`. The gradient. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 shrinkage regularization. Must be a scalar. l2_shrinkage: A `Tensor`. Must have the same type as `var`. lr_power: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_ftrl_v2 op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyFtrlV2", var=var, accum=accum, linear=linear, grad=grad, lr=lr, l1=l1, l2=l2, l2_shrinkage=l2_shrinkage, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking"), "multiply_linear_by_lr", _op._get_attr_bool("multiply_linear_by_lr")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyFtrlV2", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyFtrlV2 = tf_export("raw_ops.ApplyFtrlV2")(_ops.to_raw_op(apply_ftrl_v2)) def apply_ftrl_v2_eager_fallback(var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power, use_locking, multiply_linear_by_lr, name, ctx): raise RuntimeError("apply_ftrl_v2 op does not support eager execution. Arg 'out' is a ref.") def apply_gradient_descent(var, alpha, delta, use_locking=False, name=None): r"""Update '*var' by subtracting 'alpha' * 'delta' from it. Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). alpha: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. delta: A `Tensor`. Must have the same type as `var`. The change. use_locking: An optional `bool`. Defaults to `False`. If `True`, the subtraction will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_gradient_descent op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyGradientDescent", var=var, alpha=alpha, delta=delta, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyGradientDescent", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyGradientDescent = tf_export("raw_ops.ApplyGradientDescent")(_ops.to_raw_op(apply_gradient_descent)) def apply_gradient_descent_eager_fallback(var, alpha, delta, use_locking, name, ctx): raise RuntimeError("apply_gradient_descent op does not support eager execution. Arg 'out' is a ref.") def apply_momentum(var, accum, lr, grad, momentum, use_locking=False, use_nesterov=False, name=None): r"""Update '*var' according to the momentum scheme. Set use_nesterov = True if you want to use Nesterov momentum. accum = accum * momentum + grad var -= lr * accum Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. momentum: A `Tensor`. Must have the same type as `var`. Momentum. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, the tensor passed to compute grad will be var - lr * momentum * accum, so in the end, the var you get is actually var - lr * momentum * accum. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_momentum op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyMomentum", var=var, accum=accum, lr=lr, grad=grad, momentum=momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking"), "use_nesterov", _op._get_attr_bool("use_nesterov")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyMomentum", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyMomentum = tf_export("raw_ops.ApplyMomentum")(_ops.to_raw_op(apply_momentum)) def apply_momentum_eager_fallback(var, accum, lr, grad, momentum, use_locking, use_nesterov, name, ctx): raise RuntimeError("apply_momentum op does not support eager execution. Arg 'out' is a ref.") def apply_power_sign(var, m, lr, logbase, sign_decay, beta, grad, use_locking=False, name=None): r"""Update '*var' according to the AddSign update. m_t <- beta1 * m_{t-1} + (1 - beta1) * g update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g variable <- variable - lr_t * update Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). m: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. logbase: A `Tensor`. Must have the same type as `var`. Must be a scalar. sign_decay: A `Tensor`. Must have the same type as `var`. Must be a scalar. beta: A `Tensor`. Must have the same type as `var`. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and m tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_power_sign op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyPowerSign", var=var, m=m, lr=lr, logbase=logbase, sign_decay=sign_decay, beta=beta, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyPowerSign", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyPowerSign = tf_export("raw_ops.ApplyPowerSign")(_ops.to_raw_op(apply_power_sign)) def apply_power_sign_eager_fallback(var, m, lr, logbase, sign_decay, beta, grad, use_locking, name, ctx): raise RuntimeError("apply_power_sign op does not support eager execution. Arg 'out' is a ref.") def apply_proximal_adagrad(var, accum, lr, l1, l2, grad, use_locking=False, name=None): r"""Update '*var' and '*accum' according to FOBOS with Adagrad learning rate. accum += grad * grad prox_v = var - lr * grad * (1 / sqrt(accum)) var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0} Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_proximal_adagrad op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyProximalAdagrad", var=var, accum=accum, lr=lr, l1=l1, l2=l2, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyProximalAdagrad", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyProximalAdagrad = tf_export("raw_ops.ApplyProximalAdagrad")(_ops.to_raw_op(apply_proximal_adagrad)) def apply_proximal_adagrad_eager_fallback(var, accum, lr, l1, l2, grad, use_locking, name, ctx): raise RuntimeError("apply_proximal_adagrad op does not support eager execution. Arg 'out' is a ref.") def apply_proximal_gradient_descent(var, alpha, l1, l2, delta, use_locking=False, name=None): r"""Update '*var' as FOBOS algorithm with fixed learning rate. prox_v = var - alpha * delta var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). alpha: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. delta: A `Tensor`. Must have the same type as `var`. The change. use_locking: An optional `bool`. Defaults to `False`. If True, the subtraction will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_proximal_gradient_descent op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyProximalGradientDescent", var=var, alpha=alpha, l1=l1, l2=l2, delta=delta, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyProximalGradientDescent", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyProximalGradientDescent = tf_export("raw_ops.ApplyProximalGradientDescent")(_ops.to_raw_op(apply_proximal_gradient_descent)) def apply_proximal_gradient_descent_eager_fallback(var, alpha, l1, l2, delta, use_locking, name, ctx): raise RuntimeError("apply_proximal_gradient_descent op does not support eager execution. Arg 'out' is a ref.") def apply_rms_prop(var, ms, mom, lr, rho, momentum, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the RMSProp algorithm. Note that in dense implementation of this algorithm, ms and mom will update even if the grad is zero, but in this sparse implementation, ms and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 Delta = learning_rate * gradient / sqrt(mean_square + epsilon) ms <- rho * ms_{t-1} + (1-rho) * grad * grad mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) var <- var - mom Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). ms: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). mom: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `var`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `var`. epsilon: A `Tensor`. Must have the same type as `var`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("apply_rms_prop op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ApplyRMSProp", var=var, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "ApplyRMSProp", _inputs_flat, _attrs, _result) _result, = _result return _result ApplyRMSProp = tf_export("raw_ops.ApplyRMSProp")(_ops.to_raw_op(apply_rms_prop)) def apply_rms_prop_eager_fallback(var, ms, mom, lr, rho, momentum, epsilon, grad, use_locking, name, ctx): raise RuntimeError("apply_rms_prop op does not support eager execution. Arg 'out' is a ref.") def resource_apply_ada_max(var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the AdaMax algorithm. m_t <- beta1 * m_{t-1} + (1 - beta1) * g v_t <- max(beta2 * v_{t-1}, abs(g)) variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon) Args: var: A `Tensor` of type `resource`. Should be from a Variable(). m: A `Tensor` of type `resource`. Should be from a Variable(). v: A `Tensor` of type `resource`. Should be from a Variable(). beta1_power: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Must be a scalar. lr: A `Tensor`. Must have the same type as `beta1_power`. Scaling factor. Must be a scalar. beta1: A `Tensor`. Must have the same type as `beta1_power`. Momentum factor. Must be a scalar. beta2: A `Tensor`. Must have the same type as `beta1_power`. Momentum factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `beta1_power`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `beta1_power`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, m, and v tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdaMax", name, var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_ada_max_eager_fallback( var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdaMax", var=var, m=m, v=v, beta1_power=beta1_power, lr=lr, beta1=beta1, beta2=beta2, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyAdaMax = tf_export("raw_ops.ResourceApplyAdaMax")(_ops.to_raw_op(resource_apply_ada_max)) def resource_apply_ada_max_eager_fallback(var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([beta1_power, lr, beta1, beta2, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (beta1_power, lr, beta1, beta2, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) m = _ops.convert_to_tensor(m, _dtypes.resource) v = _ops.convert_to_tensor(v, _dtypes.resource) _inputs_flat = [var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyAdaMax", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_adadelta(var, accum, accum_update, lr, rho, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the adadelta scheme. accum = rho() * accum + (1 - rho()) * grad.square(); update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad; update_accum = rho() * update_accum + (1 - rho()) * update.square(); var -= update; Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). accum_update: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `lr`. Decay factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `lr`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var, accum and update_accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdadelta", name, var, accum, accum_update, lr, rho, epsilon, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_adadelta_eager_fallback( var, accum, accum_update, lr, rho, epsilon, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdadelta", var=var, accum=accum, accum_update=accum_update, lr=lr, rho=rho, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyAdadelta = tf_export("raw_ops.ResourceApplyAdadelta")(_ops.to_raw_op(resource_apply_adadelta)) def resource_apply_adadelta_eager_fallback(var, accum, accum_update, lr, rho, epsilon, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, rho, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, rho, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) accum_update = _ops.convert_to_tensor(accum_update, _dtypes.resource) _inputs_flat = [var, accum, accum_update, lr, rho, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyAdadelta", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_adagrad(var, accum, lr, grad, use_locking=False, update_slots=True, name=None): r"""Update '*var' according to the adagrad scheme. accum += grad * grad var -= lr * grad * (1 / sqrt(accum)) Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdagrad", name, var, accum, lr, grad, "use_locking", use_locking, "update_slots", update_slots) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_adagrad_eager_fallback( var, accum, lr, grad, use_locking=use_locking, update_slots=update_slots, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdagrad", var=var, accum=accum, lr=lr, grad=grad, use_locking=use_locking, update_slots=update_slots, name=name) return _op ResourceApplyAdagrad = tf_export("raw_ops.ResourceApplyAdagrad")(_ops.to_raw_op(resource_apply_adagrad)) def resource_apply_adagrad_eager_fallback(var, accum, lr, grad, use_locking, update_slots, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, grad] _attrs = ("T", _attr_T, "use_locking", use_locking, "update_slots", update_slots) _result = _execute.execute(b"ResourceApplyAdagrad", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_adagrad_da(var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step, use_locking=False, name=None): r"""Update '*var' according to the proximal adagrad scheme. Args: var: A `Tensor` of type `resource`. Should be from a Variable(). gradient_accumulator: A `Tensor` of type `resource`. Should be from a Variable(). gradient_squared_accumulator: A `Tensor` of type `resource`. Should be from a Variable(). grad: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. The gradient. lr: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `grad`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `grad`. L2 regularization. Must be a scalar. global_step: A `Tensor` of type `int64`. Training step number. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdagradDA", name, var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_adagrad_da_eager_fallback( var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdagradDA", var=var, gradient_accumulator=gradient_accumulator, gradient_squared_accumulator=gradient_squared_accumulator, grad=grad, lr=lr, l1=l1, l2=l2, global_step=global_step, use_locking=use_locking, name=name) return _op ResourceApplyAdagradDA = tf_export("raw_ops.ResourceApplyAdagradDA")(_ops.to_raw_op(resource_apply_adagrad_da)) def resource_apply_adagrad_da_eager_fallback(var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([grad, lr, l1, l2], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (grad, lr, l1, l2) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) gradient_accumulator = _ops.convert_to_tensor(gradient_accumulator, _dtypes.resource) gradient_squared_accumulator = _ops.convert_to_tensor(gradient_squared_accumulator, _dtypes.resource) global_step = _ops.convert_to_tensor(global_step, _dtypes.int64) _inputs_flat = [var, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyAdagradDA", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_adagrad_v2(var, accum, lr, epsilon, grad, use_locking=False, update_slots=True, name=None): r"""Update '*var' according to the adagrad scheme. accum += grad * grad var -= lr * grad * (1 / (sqrt(accum) + epsilon)) Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `lr`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdagradV2", name, var, accum, lr, epsilon, grad, "use_locking", use_locking, "update_slots", update_slots) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_adagrad_v2_eager_fallback( var, accum, lr, epsilon, grad, use_locking=use_locking, update_slots=update_slots, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdagradV2", var=var, accum=accum, lr=lr, epsilon=epsilon, grad=grad, use_locking=use_locking, update_slots=update_slots, name=name) return _op ResourceApplyAdagradV2 = tf_export("raw_ops.ResourceApplyAdagradV2")(_ops.to_raw_op(resource_apply_adagrad_v2)) def resource_apply_adagrad_v2_eager_fallback(var, accum, lr, epsilon, grad, use_locking, update_slots, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking, "update_slots", update_slots) _result = _execute.execute(b"ResourceApplyAdagradV2", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_adam(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking=False, use_nesterov=False, name=None): r"""Update '*var' according to the Adam algorithm. $$\text{lr}_t := \mathrm{learning_rate} * \sqrt{1 - \beta_2^t} / (1 - \beta_1^t)$$ $$m_t := \beta_1 * m_{t-1} + (1 - \beta_1) * g$$ $$v_t := \beta_2 * v_{t-1} + (1 - \beta_2) * g * g$$ $$\text{variable} := \text{variable} - \text{lr}_t * m_t / (\sqrt{v_t} + \epsilon)$$ Args: var: A `Tensor` of type `resource`. Should be from a Variable(). m: A `Tensor` of type `resource`. Should be from a Variable(). v: A `Tensor` of type `resource`. Should be from a Variable(). beta1_power: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Must be a scalar. beta2_power: A `Tensor`. Must have the same type as `beta1_power`. Must be a scalar. lr: A `Tensor`. Must have the same type as `beta1_power`. Scaling factor. Must be a scalar. beta1: A `Tensor`. Must have the same type as `beta1_power`. Momentum factor. Must be a scalar. beta2: A `Tensor`. Must have the same type as `beta1_power`. Momentum factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `beta1_power`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `beta1_power`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, m, and v tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, uses the nesterov update. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdam", name, var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, "use_locking", use_locking, "use_nesterov", use_nesterov) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_adam_eager_fallback( var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking=use_locking, use_nesterov=use_nesterov, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdam", var=var, m=m, v=v, beta1_power=beta1_power, beta2_power=beta2_power, lr=lr, beta1=beta1, beta2=beta2, epsilon=epsilon, grad=grad, use_locking=use_locking, use_nesterov=use_nesterov, name=name) return _op ResourceApplyAdam = tf_export("raw_ops.ResourceApplyAdam")(_ops.to_raw_op(resource_apply_adam)) def resource_apply_adam_eager_fallback(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking, use_nesterov, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _attr_T, _inputs_T = _execute.args_to_matching_eager([beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) m = _ops.convert_to_tensor(m, _dtypes.resource) v = _ops.convert_to_tensor(v, _dtypes.resource) _inputs_flat = [var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking, "use_nesterov", use_nesterov) _result = _execute.execute(b"ResourceApplyAdam", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_adam_with_amsgrad(var, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the Adam algorithm. $$\text{lr}_t := \mathrm{learning_rate} * \sqrt{1 - \beta_2^t} / (1 - \beta_1^t)$$ $$m_t := \beta_1 * m_{t-1} + (1 - \beta_1) * g$$ $$v_t := \beta_2 * v_{t-1} + (1 - \beta_2) * g * g$$ $$\hat{v}_t := max{\hat{v}_{t-1}, v_t}$$ $$\text{variable} := \text{variable} - \text{lr}_t * m_t / (\sqrt{\hat{v}_t} + \epsilon)$$ Args: var: A `Tensor` of type `resource`. Should be from a Variable(). m: A `Tensor` of type `resource`. Should be from a Variable(). v: A `Tensor` of type `resource`. Should be from a Variable(). vhat: A `Tensor` of type `resource`. Should be from a Variable(). beta1_power: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Must be a scalar. beta2_power: A `Tensor`. Must have the same type as `beta1_power`. Must be a scalar. lr: A `Tensor`. Must have the same type as `beta1_power`. Scaling factor. Must be a scalar. beta1: A `Tensor`. Must have the same type as `beta1_power`. Momentum factor. Must be a scalar. beta2: A `Tensor`. Must have the same type as `beta1_power`. Momentum factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `beta1_power`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `beta1_power`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, m, and v tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAdamWithAmsgrad", name, var, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_adam_with_amsgrad_eager_fallback( var, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAdamWithAmsgrad", var=var, m=m, v=v, vhat=vhat, beta1_power=beta1_power, beta2_power=beta2_power, lr=lr, beta1=beta1, beta2=beta2, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyAdamWithAmsgrad = tf_export("raw_ops.ResourceApplyAdamWithAmsgrad")(_ops.to_raw_op(resource_apply_adam_with_amsgrad)) def resource_apply_adam_with_amsgrad_eager_fallback(var, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) m = _ops.convert_to_tensor(m, _dtypes.resource) v = _ops.convert_to_tensor(v, _dtypes.resource) vhat = _ops.convert_to_tensor(vhat, _dtypes.resource) _inputs_flat = [var, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyAdamWithAmsgrad", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_add_sign(var, m, lr, alpha, sign_decay, beta, grad, use_locking=False, name=None): r"""Update '*var' according to the AddSign update. m_t <- beta1 * m_{t-1} + (1 - beta1) * g update <- (alpha + sign_decay * sign(g) *sign(m)) * g variable <- variable - lr_t * update Args: var: A `Tensor` of type `resource`. Should be from a Variable(). m: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. alpha: A `Tensor`. Must have the same type as `lr`. Must be a scalar. sign_decay: A `Tensor`. Must have the same type as `lr`. Must be a scalar. beta: A `Tensor`. Must have the same type as `lr`. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and m tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyAddSign", name, var, m, lr, alpha, sign_decay, beta, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_add_sign_eager_fallback( var, m, lr, alpha, sign_decay, beta, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyAddSign", var=var, m=m, lr=lr, alpha=alpha, sign_decay=sign_decay, beta=beta, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyAddSign = tf_export("raw_ops.ResourceApplyAddSign")(_ops.to_raw_op(resource_apply_add_sign)) def resource_apply_add_sign_eager_fallback(var, m, lr, alpha, sign_decay, beta, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, alpha, sign_decay, beta, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, alpha, sign_decay, beta, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) m = _ops.convert_to_tensor(m, _dtypes.resource) _inputs_flat = [var, m, lr, alpha, sign_decay, beta, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyAddSign", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_centered_rms_prop(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the centered RMSProp algorithm. The centered RMSProp algorithm uses an estimate of the centered second moment (i.e., the variance) for normalization, as opposed to regular RMSProp, which uses the (uncentered) second moment. This often helps with training, but is slightly more expensive in terms of computation and memory. Note that in dense implementation of this algorithm, mg, ms, and mom will update even if the grad is zero, but in this sparse implementation, mg, ms, and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 mean_grad = decay * mean_grad + (1-decay) * gradient Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2) mg <- rho * mg_{t-1} + (1-rho) * grad ms <- rho * ms_{t-1} + (1-rho) * grad * grad mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon) var <- var - mom Args: var: A `Tensor` of type `resource`. Should be from a Variable(). mg: A `Tensor` of type `resource`. Should be from a Variable(). ms: A `Tensor` of type `resource`. Should be from a Variable(). mom: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `lr`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `lr`. Momentum Scale. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `lr`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, mg, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyCenteredRMSProp", name, var, mg, ms, mom, lr, rho, momentum, epsilon, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_centered_rms_prop_eager_fallback( var, mg, ms, mom, lr, rho, momentum, epsilon, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyCenteredRMSProp", var=var, mg=mg, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyCenteredRMSProp = tf_export("raw_ops.ResourceApplyCenteredRMSProp")(_ops.to_raw_op(resource_apply_centered_rms_prop)) def resource_apply_centered_rms_prop_eager_fallback(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, rho, momentum, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, rho, momentum, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) mg = _ops.convert_to_tensor(mg, _dtypes.resource) ms = _ops.convert_to_tensor(ms, _dtypes.resource) mom = _ops.convert_to_tensor(mom, _dtypes.resource) _inputs_flat = [var, mg, ms, mom, lr, rho, momentum, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyCenteredRMSProp", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_ftrl(var, accum, linear, grad, lr, l1, l2, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update '*var' according to the Ftrl-proximal scheme. accum_new = accum + grad * grad linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). linear: A `Tensor` of type `resource`. Should be from a Variable(). grad: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. The gradient. lr: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `grad`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `grad`. L2 regularization. Must be a scalar. lr_power: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyFtrl", name, var, accum, linear, grad, lr, l1, l2, lr_power, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_ftrl_eager_fallback( var, accum, linear, grad, lr, l1, l2, lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyFtrl", var=var, accum=accum, linear=linear, grad=grad, lr=lr, l1=l1, l2=l2, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) return _op ResourceApplyFtrl = tf_export("raw_ops.ResourceApplyFtrl")(_ops.to_raw_op(resource_apply_ftrl)) def resource_apply_ftrl_eager_fallback(var, accum, linear, grad, lr, l1, l2, lr_power, use_locking, multiply_linear_by_lr, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _attr_T, _inputs_T = _execute.args_to_matching_eager([grad, lr, l1, l2, lr_power], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (grad, lr, l1, l2, lr_power) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) linear = _ops.convert_to_tensor(linear, _dtypes.resource) _inputs_flat = [var, accum, linear, grad, lr, l1, l2, lr_power] _attrs = ("T", _attr_T, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) _result = _execute.execute(b"ResourceApplyFtrl", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_ftrl_v2(var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update '*var' according to the Ftrl-proximal scheme. grad_with_shrinkage = grad + 2 * l2_shrinkage * var accum_new = accum + grad_with_shrinkage * grad_with_shrinkage linear += grad_with_shrinkage + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). linear: A `Tensor` of type `resource`. Should be from a Variable(). grad: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. The gradient. lr: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `grad`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `grad`. L2 shrinkage regularization. Must be a scalar. l2_shrinkage: A `Tensor`. Must have the same type as `grad`. lr_power: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyFtrlV2", name, var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_ftrl_v2_eager_fallback( var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyFtrlV2", var=var, accum=accum, linear=linear, grad=grad, lr=lr, l1=l1, l2=l2, l2_shrinkage=l2_shrinkage, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) return _op ResourceApplyFtrlV2 = tf_export("raw_ops.ResourceApplyFtrlV2")(_ops.to_raw_op(resource_apply_ftrl_v2)) def resource_apply_ftrl_v2_eager_fallback(var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power, use_locking, multiply_linear_by_lr, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _attr_T, _inputs_T = _execute.args_to_matching_eager([grad, lr, l1, l2, l2_shrinkage, lr_power], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (grad, lr, l1, l2, l2_shrinkage, lr_power) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) linear = _ops.convert_to_tensor(linear, _dtypes.resource) _inputs_flat = [var, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power] _attrs = ("T", _attr_T, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) _result = _execute.execute(b"ResourceApplyFtrlV2", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_gradient_descent(var, alpha, delta, use_locking=False, name=None): r"""Update '*var' by subtracting 'alpha' * 'delta' from it. Args: var: A `Tensor` of type `resource`. Should be from a Variable(). alpha: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. delta: A `Tensor`. Must have the same type as `alpha`. The change. use_locking: An optional `bool`. Defaults to `False`. If `True`, the subtraction will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyGradientDescent", name, var, alpha, delta, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_gradient_descent_eager_fallback( var, alpha, delta, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyGradientDescent", var=var, alpha=alpha, delta=delta, use_locking=use_locking, name=name) return _op ResourceApplyGradientDescent = tf_export("raw_ops.ResourceApplyGradientDescent")(_ops.to_raw_op(resource_apply_gradient_descent)) def resource_apply_gradient_descent_eager_fallback(var, alpha, delta, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([alpha, delta], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (alpha, delta) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) _inputs_flat = [var, alpha, delta] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyGradientDescent", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_keras_momentum(var, accum, lr, grad, momentum, use_locking=False, use_nesterov=False, name=None): r"""Update '*var' according to the momentum scheme. Set use_nesterov = True if you want to use Nesterov momentum. accum = accum * momentum - lr * grad var += accum Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. momentum: A `Tensor`. Must have the same type as `lr`. Momentum. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, the tensor passed to compute grad will be var + momentum * accum, so in the end, the var you get is actually var + momentum * accum. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyKerasMomentum", name, var, accum, lr, grad, momentum, "use_locking", use_locking, "use_nesterov", use_nesterov) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_keras_momentum_eager_fallback( var, accum, lr, grad, momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyKerasMomentum", var=var, accum=accum, lr=lr, grad=grad, momentum=momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name) return _op ResourceApplyKerasMomentum = tf_export("raw_ops.ResourceApplyKerasMomentum")(_ops.to_raw_op(resource_apply_keras_momentum)) def resource_apply_keras_momentum_eager_fallback(var, accum, lr, grad, momentum, use_locking, use_nesterov, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, grad, momentum], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, grad, momentum) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, grad, momentum] _attrs = ("T", _attr_T, "use_locking", use_locking, "use_nesterov", use_nesterov) _result = _execute.execute(b"ResourceApplyKerasMomentum", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_momentum(var, accum, lr, grad, momentum, use_locking=False, use_nesterov=False, name=None): r"""Update '*var' according to the momentum scheme. Set use_nesterov = True if you want to use Nesterov momentum. accum = accum * momentum + grad var -= lr * accum Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. momentum: A `Tensor`. Must have the same type as `lr`. Momentum. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, the tensor passed to compute grad will be var - lr * momentum * accum, so in the end, the var you get is actually var - lr * momentum * accum. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyMomentum", name, var, accum, lr, grad, momentum, "use_locking", use_locking, "use_nesterov", use_nesterov) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_momentum_eager_fallback( var, accum, lr, grad, momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyMomentum", var=var, accum=accum, lr=lr, grad=grad, momentum=momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name) return _op ResourceApplyMomentum = tf_export("raw_ops.ResourceApplyMomentum")(_ops.to_raw_op(resource_apply_momentum)) def resource_apply_momentum_eager_fallback(var, accum, lr, grad, momentum, use_locking, use_nesterov, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, grad, momentum], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, grad, momentum) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, grad, momentum] _attrs = ("T", _attr_T, "use_locking", use_locking, "use_nesterov", use_nesterov) _result = _execute.execute(b"ResourceApplyMomentum", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_power_sign(var, m, lr, logbase, sign_decay, beta, grad, use_locking=False, name=None): r"""Update '*var' according to the AddSign update. m_t <- beta1 * m_{t-1} + (1 - beta1) * g update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g variable <- variable - lr_t * update Args: var: A `Tensor` of type `resource`. Should be from a Variable(). m: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. logbase: A `Tensor`. Must have the same type as `lr`. Must be a scalar. sign_decay: A `Tensor`. Must have the same type as `lr`. Must be a scalar. beta: A `Tensor`. Must have the same type as `lr`. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and m tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyPowerSign", name, var, m, lr, logbase, sign_decay, beta, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_power_sign_eager_fallback( var, m, lr, logbase, sign_decay, beta, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyPowerSign", var=var, m=m, lr=lr, logbase=logbase, sign_decay=sign_decay, beta=beta, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyPowerSign = tf_export("raw_ops.ResourceApplyPowerSign")(_ops.to_raw_op(resource_apply_power_sign)) def resource_apply_power_sign_eager_fallback(var, m, lr, logbase, sign_decay, beta, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, logbase, sign_decay, beta, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, logbase, sign_decay, beta, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) m = _ops.convert_to_tensor(m, _dtypes.resource) _inputs_flat = [var, m, lr, logbase, sign_decay, beta, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyPowerSign", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_proximal_adagrad(var, accum, lr, l1, l2, grad, use_locking=False, name=None): r"""Update '*var' and '*accum' according to FOBOS with Adagrad learning rate. accum += grad * grad prox_v = var - lr * grad * (1 / sqrt(accum)) var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0} Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `lr`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `lr`. L2 regularization. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyProximalAdagrad", name, var, accum, lr, l1, l2, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_proximal_adagrad_eager_fallback( var, accum, lr, l1, l2, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyProximalAdagrad", var=var, accum=accum, lr=lr, l1=l1, l2=l2, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyProximalAdagrad = tf_export("raw_ops.ResourceApplyProximalAdagrad")(_ops.to_raw_op(resource_apply_proximal_adagrad)) def resource_apply_proximal_adagrad_eager_fallback(var, accum, lr, l1, l2, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, l1, l2, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, l1, l2, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, l1, l2, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyProximalAdagrad", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_proximal_gradient_descent(var, alpha, l1, l2, delta, use_locking=False, name=None): r"""Update '*var' as FOBOS algorithm with fixed learning rate. prox_v = var - alpha * delta var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} Args: var: A `Tensor` of type `resource`. Should be from a Variable(). alpha: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `alpha`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `alpha`. L2 regularization. Must be a scalar. delta: A `Tensor`. Must have the same type as `alpha`. The change. use_locking: An optional `bool`. Defaults to `False`. If True, the subtraction will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyProximalGradientDescent", name, var, alpha, l1, l2, delta, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_proximal_gradient_descent_eager_fallback( var, alpha, l1, l2, delta, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyProximalGradientDescent", var=var, alpha=alpha, l1=l1, l2=l2, delta=delta, use_locking=use_locking, name=name) return _op ResourceApplyProximalGradientDescent = tf_export("raw_ops.ResourceApplyProximalGradientDescent")(_ops.to_raw_op(resource_apply_proximal_gradient_descent)) def resource_apply_proximal_gradient_descent_eager_fallback(var, alpha, l1, l2, delta, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([alpha, l1, l2, delta], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (alpha, l1, l2, delta) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) _inputs_flat = [var, alpha, l1, l2, delta] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyProximalGradientDescent", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_apply_rms_prop(var, ms, mom, lr, rho, momentum, epsilon, grad, use_locking=False, name=None): r"""Update '*var' according to the RMSProp algorithm. Note that in dense implementation of this algorithm, ms and mom will update even if the grad is zero, but in this sparse implementation, ms and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 Delta = learning_rate * gradient / sqrt(mean_square + epsilon) ms <- rho * ms_{t-1} + (1-rho) * grad * grad mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) var <- var - mom Args: var: A `Tensor` of type `resource`. Should be from a Variable(). ms: A `Tensor` of type `resource`. Should be from a Variable(). mom: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `lr`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `lr`. epsilon: A `Tensor`. Must have the same type as `lr`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceApplyRMSProp", name, var, ms, mom, lr, rho, momentum, epsilon, grad, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_apply_rms_prop_eager_fallback( var, ms, mom, lr, rho, momentum, epsilon, grad, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceApplyRMSProp", var=var, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, use_locking=use_locking, name=name) return _op ResourceApplyRMSProp = tf_export("raw_ops.ResourceApplyRMSProp")(_ops.to_raw_op(resource_apply_rms_prop)) def resource_apply_rms_prop_eager_fallback(var, ms, mom, lr, rho, momentum, epsilon, grad, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, rho, momentum, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, rho, momentum, epsilon, grad) = _inputs_T var = _ops.convert_to_tensor(var, _dtypes.resource) ms = _ops.convert_to_tensor(ms, _dtypes.resource) mom = _ops.convert_to_tensor(mom, _dtypes.resource) _inputs_flat = [var, ms, mom, lr, rho, momentum, epsilon, grad] _attrs = ("T", _attr_T, "use_locking", use_locking) _result = _execute.execute(b"ResourceApplyRMSProp", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_adadelta(var, accum, accum_update, lr, rho, epsilon, grad, indices, use_locking=False, name=None): r"""var: Should be from a Variable(). Args: var: A `Tensor` of type `resource`. accum: A `Tensor` of type `resource`. Should be from a Variable(). accum_update: A `Tensor` of type `resource`. : Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Learning rate. Must be a scalar. rho: A `Tensor`. Must have the same type as `lr`. Decay factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `lr`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyAdadelta", name, var, accum, accum_update, lr, rho, epsilon, grad, indices, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_adadelta_eager_fallback( var, accum, accum_update, lr, rho, epsilon, grad, indices, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyAdadelta", var=var, accum=accum, accum_update=accum_update, lr=lr, rho=rho, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, name=name) return _op ResourceSparseApplyAdadelta = tf_export("raw_ops.ResourceSparseApplyAdadelta")(_ops.to_raw_op(resource_sparse_apply_adadelta)) def resource_sparse_apply_adadelta_eager_fallback(var, accum, accum_update, lr, rho, epsilon, grad, indices, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, rho, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, rho, epsilon, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) accum_update = _ops.convert_to_tensor(accum_update, _dtypes.resource) _inputs_flat = [var, accum, accum_update, lr, rho, epsilon, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking) _result = _execute.execute(b"ResourceSparseApplyAdadelta", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_adagrad(var, accum, lr, grad, indices, use_locking=False, update_slots=True, name=None): r"""Update relevant entries in '*var' and '*accum' according to the adagrad scheme. That is for rows we have grad for, we update var and accum as follows: accum += grad * grad var -= lr * grad * (1 / sqrt(accum)) Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Learning rate. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyAdagrad", name, var, accum, lr, grad, indices, "use_locking", use_locking, "update_slots", update_slots) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_adagrad_eager_fallback( var, accum, lr, grad, indices, use_locking=use_locking, update_slots=update_slots, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyAdagrad", var=var, accum=accum, lr=lr, grad=grad, indices=indices, use_locking=use_locking, update_slots=update_slots, name=name) return _op ResourceSparseApplyAdagrad = tf_export("raw_ops.ResourceSparseApplyAdagrad")(_ops.to_raw_op(resource_sparse_apply_adagrad)) def resource_sparse_apply_adagrad_eager_fallback(var, accum, lr, grad, indices, use_locking, update_slots, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking, "update_slots", update_slots) _result = _execute.execute(b"ResourceSparseApplyAdagrad", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_adagrad_da(var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step, use_locking=False, name=None): r"""Update entries in '*var' and '*accum' according to the proximal adagrad scheme. Args: var: A `Tensor` of type `resource`. Should be from a Variable(). gradient_accumulator: A `Tensor` of type `resource`. Should be from a Variable(). gradient_squared_accumulator: A `Tensor` of type `resource`. Should be from a Variable(). grad: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. lr: A `Tensor`. Must have the same type as `grad`. Learning rate. Must be a scalar. l1: A `Tensor`. Must have the same type as `grad`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `grad`. L2 regularization. Must be a scalar. global_step: A `Tensor` of type `int64`. Training step number. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyAdagradDA", name, var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_adagrad_da_eager_fallback( var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyAdagradDA", var=var, gradient_accumulator=gradient_accumulator, gradient_squared_accumulator=gradient_squared_accumulator, grad=grad, indices=indices, lr=lr, l1=l1, l2=l2, global_step=global_step, use_locking=use_locking, name=name) return _op ResourceSparseApplyAdagradDA = tf_export("raw_ops.ResourceSparseApplyAdagradDA")(_ops.to_raw_op(resource_sparse_apply_adagrad_da)) def resource_sparse_apply_adagrad_da_eager_fallback(var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([grad, lr, l1, l2], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (grad, lr, l1, l2) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) gradient_accumulator = _ops.convert_to_tensor(gradient_accumulator, _dtypes.resource) gradient_squared_accumulator = _ops.convert_to_tensor(gradient_squared_accumulator, _dtypes.resource) global_step = _ops.convert_to_tensor(global_step, _dtypes.int64) _inputs_flat = [var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking) _result = _execute.execute(b"ResourceSparseApplyAdagradDA", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_adagrad_v2(var, accum, lr, epsilon, grad, indices, use_locking=False, update_slots=True, name=None): r"""Update relevant entries in '*var' and '*accum' according to the adagrad scheme. That is for rows we have grad for, we update var and accum as follows: accum += grad * grad var -= lr * grad * (1 / sqrt(accum)) Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Learning rate. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `lr`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyAdagradV2", name, var, accum, lr, epsilon, grad, indices, "use_locking", use_locking, "update_slots", update_slots) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_adagrad_v2_eager_fallback( var, accum, lr, epsilon, grad, indices, use_locking=use_locking, update_slots=update_slots, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyAdagradV2", var=var, accum=accum, lr=lr, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, update_slots=update_slots, name=name) return _op ResourceSparseApplyAdagradV2 = tf_export("raw_ops.ResourceSparseApplyAdagradV2")(_ops.to_raw_op(resource_sparse_apply_adagrad_v2)) def resource_sparse_apply_adagrad_v2_eager_fallback(var, accum, lr, epsilon, grad, indices, use_locking, update_slots, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, epsilon, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, epsilon, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking, "update_slots", update_slots) _result = _execute.execute(b"ResourceSparseApplyAdagradV2", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_centered_rms_prop(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking=False, name=None): r"""Update '*var' according to the centered RMSProp algorithm. The centered RMSProp algorithm uses an estimate of the centered second moment (i.e., the variance) for normalization, as opposed to regular RMSProp, which uses the (uncentered) second moment. This often helps with training, but is slightly more expensive in terms of computation and memory. Note that in dense implementation of this algorithm, mg, ms, and mom will update even if the grad is zero, but in this sparse implementation, mg, ms, and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 mean_grad = decay * mean_grad + (1-decay) * gradient Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2) ms <- rho * ms_{t-1} + (1-rho) * grad * grad mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) var <- var - mom Args: var: A `Tensor` of type `resource`. Should be from a Variable(). mg: A `Tensor` of type `resource`. Should be from a Variable(). ms: A `Tensor` of type `resource`. Should be from a Variable(). mom: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `lr`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `lr`. epsilon: A `Tensor`. Must have the same type as `lr`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var, ms and mom. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, mg, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyCenteredRMSProp", name, var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_centered_rms_prop_eager_fallback( var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyCenteredRMSProp", var=var, mg=mg, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, name=name) return _op ResourceSparseApplyCenteredRMSProp = tf_export("raw_ops.ResourceSparseApplyCenteredRMSProp")(_ops.to_raw_op(resource_sparse_apply_centered_rms_prop)) def resource_sparse_apply_centered_rms_prop_eager_fallback(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, rho, momentum, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, rho, momentum, epsilon, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) mg = _ops.convert_to_tensor(mg, _dtypes.resource) ms = _ops.convert_to_tensor(ms, _dtypes.resource) mom = _ops.convert_to_tensor(mom, _dtypes.resource) _inputs_flat = [var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking) _result = _execute.execute(b"ResourceSparseApplyCenteredRMSProp", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_ftrl(var, accum, linear, grad, indices, lr, l1, l2, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update relevant entries in '*var' according to the Ftrl-proximal scheme. That is for rows we have grad for, we update var, accum and linear as follows: accum_new = accum + grad * grad linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). linear: A `Tensor` of type `resource`. Should be from a Variable(). grad: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. lr: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `grad`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `grad`. L2 regularization. Must be a scalar. lr_power: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyFtrl", name, var, accum, linear, grad, indices, lr, l1, l2, lr_power, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_ftrl_eager_fallback( var, accum, linear, grad, indices, lr, l1, l2, lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyFtrl", var=var, accum=accum, linear=linear, grad=grad, indices=indices, lr=lr, l1=l1, l2=l2, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) return _op ResourceSparseApplyFtrl = tf_export("raw_ops.ResourceSparseApplyFtrl")(_ops.to_raw_op(resource_sparse_apply_ftrl)) def resource_sparse_apply_ftrl_eager_fallback(var, accum, linear, grad, indices, lr, l1, l2, lr_power, use_locking, multiply_linear_by_lr, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _attr_T, _inputs_T = _execute.args_to_matching_eager([grad, lr, l1, l2, lr_power], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (grad, lr, l1, l2, lr_power) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) linear = _ops.convert_to_tensor(linear, _dtypes.resource) _inputs_flat = [var, accum, linear, grad, indices, lr, l1, l2, lr_power] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) _result = _execute.execute(b"ResourceSparseApplyFtrl", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_ftrl_v2(var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update relevant entries in '*var' according to the Ftrl-proximal scheme. That is for rows we have grad for, we update var, accum and linear as follows: grad_with_shrinkage = grad + 2 * l2_shrinkage * var accum_new = accum + grad_with_shrinkage * grad_with_shrinkage linear += grad_with_shrinkage + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). linear: A `Tensor` of type `resource`. Should be from a Variable(). grad: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. lr: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `grad`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `grad`. L2 shrinkage regularization. Must be a scalar. l2_shrinkage: A `Tensor`. Must have the same type as `grad`. lr_power: A `Tensor`. Must have the same type as `grad`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyFtrlV2", name, var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_ftrl_v2_eager_fallback( var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyFtrlV2", var=var, accum=accum, linear=linear, grad=grad, indices=indices, lr=lr, l1=l1, l2=l2, l2_shrinkage=l2_shrinkage, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) return _op ResourceSparseApplyFtrlV2 = tf_export("raw_ops.ResourceSparseApplyFtrlV2")(_ops.to_raw_op(resource_sparse_apply_ftrl_v2)) def resource_sparse_apply_ftrl_v2_eager_fallback(var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power, use_locking, multiply_linear_by_lr, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _attr_T, _inputs_T = _execute.args_to_matching_eager([grad, lr, l1, l2, l2_shrinkage, lr_power], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (grad, lr, l1, l2, l2_shrinkage, lr_power) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) linear = _ops.convert_to_tensor(linear, _dtypes.resource) _inputs_flat = [var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking, "multiply_linear_by_lr", multiply_linear_by_lr) _result = _execute.execute(b"ResourceSparseApplyFtrlV2", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_keras_momentum(var, accum, lr, grad, indices, momentum, use_locking=False, use_nesterov=False, name=None): r"""Update relevant entries in '*var' and '*accum' according to the momentum scheme. Set use_nesterov = True if you want to use Nesterov momentum. That is for rows we have grad for, we update var and accum as follows: accum = accum * momentum - lr * grad var += accum Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Learning rate. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. momentum: A `Tensor`. Must have the same type as `lr`. Momentum. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, the tensor passed to compute grad will be var + momentum * accum, so in the end, the var you get is actually var + momentum * accum. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyKerasMomentum", name, var, accum, lr, grad, indices, momentum, "use_locking", use_locking, "use_nesterov", use_nesterov) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_keras_momentum_eager_fallback( var, accum, lr, grad, indices, momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyKerasMomentum", var=var, accum=accum, lr=lr, grad=grad, indices=indices, momentum=momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name) return _op ResourceSparseApplyKerasMomentum = tf_export("raw_ops.ResourceSparseApplyKerasMomentum")(_ops.to_raw_op(resource_sparse_apply_keras_momentum)) def resource_sparse_apply_keras_momentum_eager_fallback(var, accum, lr, grad, indices, momentum, use_locking, use_nesterov, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, grad, momentum], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, grad, momentum) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, grad, indices, momentum] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking, "use_nesterov", use_nesterov) _result = _execute.execute(b"ResourceSparseApplyKerasMomentum", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_momentum(var, accum, lr, grad, indices, momentum, use_locking=False, use_nesterov=False, name=None): r"""Update relevant entries in '*var' and '*accum' according to the momentum scheme. Set use_nesterov = True if you want to use Nesterov momentum. That is for rows we have grad for, we update var and accum as follows: accum = accum * momentum + grad var -= lr * accum Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Learning rate. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. momentum: A `Tensor`. Must have the same type as `lr`. Momentum. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, the tensor passed to compute grad will be var - lr * momentum * accum, so in the end, the var you get is actually var - lr * momentum * accum. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyMomentum", name, var, accum, lr, grad, indices, momentum, "use_locking", use_locking, "use_nesterov", use_nesterov) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_momentum_eager_fallback( var, accum, lr, grad, indices, momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyMomentum", var=var, accum=accum, lr=lr, grad=grad, indices=indices, momentum=momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name) return _op ResourceSparseApplyMomentum = tf_export("raw_ops.ResourceSparseApplyMomentum")(_ops.to_raw_op(resource_sparse_apply_momentum)) def resource_sparse_apply_momentum_eager_fallback(var, accum, lr, grad, indices, momentum, use_locking, use_nesterov, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, grad, momentum], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, grad, momentum) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, grad, indices, momentum] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking, "use_nesterov", use_nesterov) _result = _execute.execute(b"ResourceSparseApplyMomentum", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_proximal_adagrad(var, accum, lr, l1, l2, grad, indices, use_locking=False, name=None): r"""Sparse update entries in '*var' and '*accum' according to FOBOS algorithm. That is for rows we have grad for, we update var and accum as follows: accum += grad * grad prox_v = var prox_v -= lr * grad * (1 / sqrt(accum)) var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0} Args: var: A `Tensor` of type `resource`. Should be from a Variable(). accum: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Learning rate. Must be a scalar. l1: A `Tensor`. Must have the same type as `lr`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `lr`. L2 regularization. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyProximalAdagrad", name, var, accum, lr, l1, l2, grad, indices, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_proximal_adagrad_eager_fallback( var, accum, lr, l1, l2, grad, indices, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyProximalAdagrad", var=var, accum=accum, lr=lr, l1=l1, l2=l2, grad=grad, indices=indices, use_locking=use_locking, name=name) return _op ResourceSparseApplyProximalAdagrad = tf_export("raw_ops.ResourceSparseApplyProximalAdagrad")(_ops.to_raw_op(resource_sparse_apply_proximal_adagrad)) def resource_sparse_apply_proximal_adagrad_eager_fallback(var, accum, lr, l1, l2, grad, indices, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, l1, l2, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, l1, l2, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) accum = _ops.convert_to_tensor(accum, _dtypes.resource) _inputs_flat = [var, accum, lr, l1, l2, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking) _result = _execute.execute(b"ResourceSparseApplyProximalAdagrad", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_proximal_gradient_descent(var, alpha, l1, l2, grad, indices, use_locking=False, name=None): r"""Sparse update '*var' as FOBOS algorithm with fixed learning rate. That is for rows we have grad for, we update var as follows: prox_v = var - alpha * grad var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} Args: var: A `Tensor` of type `resource`. Should be from a Variable(). alpha: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `alpha`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `alpha`. L2 regularization. Must be a scalar. grad: A `Tensor`. Must have the same type as `alpha`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If True, the subtraction will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyProximalGradientDescent", name, var, alpha, l1, l2, grad, indices, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_proximal_gradient_descent_eager_fallback( var, alpha, l1, l2, grad, indices, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyProximalGradientDescent", var=var, alpha=alpha, l1=l1, l2=l2, grad=grad, indices=indices, use_locking=use_locking, name=name) return _op ResourceSparseApplyProximalGradientDescent = tf_export("raw_ops.ResourceSparseApplyProximalGradientDescent")(_ops.to_raw_op(resource_sparse_apply_proximal_gradient_descent)) def resource_sparse_apply_proximal_gradient_descent_eager_fallback(var, alpha, l1, l2, grad, indices, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([alpha, l1, l2, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (alpha, l1, l2, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) _inputs_flat = [var, alpha, l1, l2, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking) _result = _execute.execute(b"ResourceSparseApplyProximalGradientDescent", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def resource_sparse_apply_rms_prop(var, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking=False, name=None): r"""Update '*var' according to the RMSProp algorithm. Note that in dense implementation of this algorithm, ms and mom will update even if the grad is zero, but in this sparse implementation, ms and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 Delta = learning_rate * gradient / sqrt(mean_square + epsilon) ms <- rho * ms_{t-1} + (1-rho) * grad * grad mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon) var <- var - mom Args: var: A `Tensor` of type `resource`. Should be from a Variable(). ms: A `Tensor` of type `resource`. Should be from a Variable(). mom: A `Tensor` of type `resource`. Should be from a Variable(). lr: A `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `lr`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `lr`. epsilon: A `Tensor`. Must have the same type as `lr`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `lr`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var, ms and mom. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: The created Operation. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: try: _result = pywrap_tfe.TFE_Py_FastPathExecute( _ctx, "ResourceSparseApplyRMSProp", name, var, ms, mom, lr, rho, momentum, epsilon, grad, indices, "use_locking", use_locking) return _result except _core._NotOkStatusException as e: _ops.raise_from_not_ok_status(e, name) except _core._FallbackException: pass try: return resource_sparse_apply_rms_prop_eager_fallback( var, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking=use_locking, name=name, ctx=_ctx) except _core._SymbolicException: pass # Add nodes to the TensorFlow graph. # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "ResourceSparseApplyRMSProp", var=var, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, name=name) return _op ResourceSparseApplyRMSProp = tf_export("raw_ops.ResourceSparseApplyRMSProp")(_ops.to_raw_op(resource_sparse_apply_rms_prop)) def resource_sparse_apply_rms_prop_eager_fallback(var, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking, name, ctx): if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _attr_T, _inputs_T = _execute.args_to_matching_eager([lr, rho, momentum, epsilon, grad], ctx, [_dtypes.float32, _dtypes.float64, _dtypes.int32, _dtypes.uint8, _dtypes.int16, _dtypes.int8, _dtypes.complex64, _dtypes.int64, _dtypes.qint8, _dtypes.quint8, _dtypes.qint32, _dtypes.bfloat16, _dtypes.uint16, _dtypes.complex128, _dtypes.half, _dtypes.uint32, _dtypes.uint64, ]) (lr, rho, momentum, epsilon, grad) = _inputs_T _attr_Tindices, (indices,) = _execute.args_to_matching_eager([indices], ctx, [_dtypes.int32, _dtypes.int64, ]) var = _ops.convert_to_tensor(var, _dtypes.resource) ms = _ops.convert_to_tensor(ms, _dtypes.resource) mom = _ops.convert_to_tensor(mom, _dtypes.resource) _inputs_flat = [var, ms, mom, lr, rho, momentum, epsilon, grad, indices] _attrs = ("T", _attr_T, "Tindices", _attr_Tindices, "use_locking", use_locking) _result = _execute.execute(b"ResourceSparseApplyRMSProp", 0, inputs=_inputs_flat, attrs=_attrs, ctx=ctx, name=name) _result = None return _result def sparse_apply_adadelta(var, accum, accum_update, lr, rho, epsilon, grad, indices, use_locking=False, name=None): r"""var: Should be from a Variable(). Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). accum_update: A mutable `Tensor`. Must have the same type as `var`. : Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Learning rate. Must be a scalar. rho: A `Tensor`. Must have the same type as `var`. Decay factor. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_adadelta op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyAdadelta", var=var, accum=accum, accum_update=accum_update, lr=lr, rho=rho, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyAdadelta", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyAdadelta = tf_export("raw_ops.SparseApplyAdadelta")(_ops.to_raw_op(sparse_apply_adadelta)) def sparse_apply_adadelta_eager_fallback(var, accum, accum_update, lr, rho, epsilon, grad, indices, use_locking, name, ctx): raise RuntimeError("sparse_apply_adadelta op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_adagrad(var, accum, lr, grad, indices, use_locking=False, update_slots=True, name=None): r"""Update relevant entries in '*var' and '*accum' according to the adagrad scheme. That is for rows we have grad for, we update var and accum as follows: $$accum += grad * grad$$ $$var -= lr * grad * (1 / sqrt(accum))$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Learning rate. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_adagrad op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyAdagrad", var=var, accum=accum, lr=lr, grad=grad, indices=indices, use_locking=use_locking, update_slots=update_slots, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking"), "update_slots", _op._get_attr_bool("update_slots")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyAdagrad", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyAdagrad = tf_export("raw_ops.SparseApplyAdagrad")(_ops.to_raw_op(sparse_apply_adagrad)) def sparse_apply_adagrad_eager_fallback(var, accum, lr, grad, indices, use_locking, update_slots, name, ctx): raise RuntimeError("sparse_apply_adagrad op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_adagrad_da(var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step, use_locking=False, name=None): r"""Update entries in '*var' and '*accum' according to the proximal adagrad scheme. Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). gradient_accumulator: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). gradient_squared_accumulator: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. lr: A `Tensor`. Must have the same type as `var`. Learning rate. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. global_step: A `Tensor` of type `int64`. Training step number. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_adagrad_da op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyAdagradDA", var=var, gradient_accumulator=gradient_accumulator, gradient_squared_accumulator=gradient_squared_accumulator, grad=grad, indices=indices, lr=lr, l1=l1, l2=l2, global_step=global_step, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyAdagradDA", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyAdagradDA = tf_export("raw_ops.SparseApplyAdagradDA")(_ops.to_raw_op(sparse_apply_adagrad_da)) def sparse_apply_adagrad_da_eager_fallback(var, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step, use_locking, name, ctx): raise RuntimeError("sparse_apply_adagrad_da op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_adagrad_v2(var, accum, lr, epsilon, grad, indices, use_locking=False, update_slots=True, name=None): r"""Update relevant entries in '*var' and '*accum' according to the adagrad scheme. That is for rows we have grad for, we update var and accum as follows: $$accum += grad * grad$$ $$var -= lr * grad * (1 / sqrt(accum))$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Learning rate. Must be a scalar. epsilon: A `Tensor`. Must have the same type as `var`. Constant factor. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. update_slots: An optional `bool`. Defaults to `True`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_adagrad_v2 op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if update_slots is None: update_slots = True update_slots = _execute.make_bool(update_slots, "update_slots") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyAdagradV2", var=var, accum=accum, lr=lr, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, update_slots=update_slots, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking"), "update_slots", _op._get_attr_bool("update_slots")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyAdagradV2", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyAdagradV2 = tf_export("raw_ops.SparseApplyAdagradV2")(_ops.to_raw_op(sparse_apply_adagrad_v2)) def sparse_apply_adagrad_v2_eager_fallback(var, accum, lr, epsilon, grad, indices, use_locking, update_slots, name, ctx): raise RuntimeError("sparse_apply_adagrad_v2 op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_centered_rms_prop(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking=False, name=None): r"""Update '*var' according to the centered RMSProp algorithm. The centered RMSProp algorithm uses an estimate of the centered second moment (i.e., the variance) for normalization, as opposed to regular RMSProp, which uses the (uncentered) second moment. This often helps with training, but is slightly more expensive in terms of computation and memory. Note that in dense implementation of this algorithm, mg, ms, and mom will update even if the grad is zero, but in this sparse implementation, mg, ms, and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 mean_grad = decay * mean_grad + (1-decay) * gradient Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2) $$ms <- rho * ms_{t-1} + (1-rho) * grad * grad$$ $$mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)$$ $$var <- var - mom$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). mg: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). ms: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). mom: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `var`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `var`. epsilon: A `Tensor`. Must have the same type as `var`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var, ms and mom. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, mg, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_centered_rms_prop op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyCenteredRMSProp", var=var, mg=mg, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyCenteredRMSProp", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyCenteredRMSProp = tf_export("raw_ops.SparseApplyCenteredRMSProp")(_ops.to_raw_op(sparse_apply_centered_rms_prop)) def sparse_apply_centered_rms_prop_eager_fallback(var, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking, name, ctx): raise RuntimeError("sparse_apply_centered_rms_prop op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_ftrl(var, accum, linear, grad, indices, lr, l1, l2, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update relevant entries in '*var' according to the Ftrl-proximal scheme. That is for rows we have grad for, we update var, accum and linear as follows: $$accum_new = accum + grad * grad$$ $$linear += grad + (accum_{new}^{-lr_{power}} - accum^{-lr_{power}} / lr * var$$ $$quadratic = 1.0 / (accum_{new}^{lr_{power}} * lr) + 2 * l2$$ $$var = (sign(linear) * l1 - linear) / quadratic\ if\ |linear| > l1\ else\ 0.0$$ $$accum = accum_{new}$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). linear: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. lr_power: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_ftrl op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyFtrl", var=var, accum=accum, linear=linear, grad=grad, indices=indices, lr=lr, l1=l1, l2=l2, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking"), "multiply_linear_by_lr", _op._get_attr_bool("multiply_linear_by_lr")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyFtrl", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyFtrl = tf_export("raw_ops.SparseApplyFtrl")(_ops.to_raw_op(sparse_apply_ftrl)) def sparse_apply_ftrl_eager_fallback(var, accum, linear, grad, indices, lr, l1, l2, lr_power, use_locking, multiply_linear_by_lr, name, ctx): raise RuntimeError("sparse_apply_ftrl op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_ftrl_v2(var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power, use_locking=False, multiply_linear_by_lr=False, name=None): r"""Update relevant entries in '*var' according to the Ftrl-proximal scheme. That is for rows we have grad for, we update var, accum and linear as follows: grad_with_shrinkage = grad + 2 * l2_shrinkage * var accum_new = accum + grad * grad linear += grad_with_shrinkage - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2 var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0 accum = accum_new Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). linear: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 shrinkage regularization. Must be a scalar. l2_shrinkage: A `Tensor`. Must have the same type as `var`. lr_power: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. multiply_linear_by_lr: An optional `bool`. Defaults to `False`. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_ftrl_v2 op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if multiply_linear_by_lr is None: multiply_linear_by_lr = False multiply_linear_by_lr = _execute.make_bool(multiply_linear_by_lr, "multiply_linear_by_lr") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyFtrlV2", var=var, accum=accum, linear=linear, grad=grad, indices=indices, lr=lr, l1=l1, l2=l2, l2_shrinkage=l2_shrinkage, lr_power=lr_power, use_locking=use_locking, multiply_linear_by_lr=multiply_linear_by_lr, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking"), "multiply_linear_by_lr", _op._get_attr_bool("multiply_linear_by_lr")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyFtrlV2", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyFtrlV2 = tf_export("raw_ops.SparseApplyFtrlV2")(_ops.to_raw_op(sparse_apply_ftrl_v2)) def sparse_apply_ftrl_v2_eager_fallback(var, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power, use_locking, multiply_linear_by_lr, name, ctx): raise RuntimeError("sparse_apply_ftrl_v2 op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_momentum(var, accum, lr, grad, indices, momentum, use_locking=False, use_nesterov=False, name=None): r"""Update relevant entries in '*var' and '*accum' according to the momentum scheme. Set use_nesterov = True if you want to use Nesterov momentum. That is for rows we have grad for, we update var and accum as follows: $$accum = accum * momentum + grad$$ $$var -= lr * accum$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Learning rate. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. momentum: A `Tensor`. Must have the same type as `var`. Momentum. Must be a scalar. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. use_nesterov: An optional `bool`. Defaults to `False`. If `True`, the tensor passed to compute grad will be var - lr * momentum * accum, so in the end, the var you get is actually var - lr * momentum * accum. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_momentum op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") if use_nesterov is None: use_nesterov = False use_nesterov = _execute.make_bool(use_nesterov, "use_nesterov") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyMomentum", var=var, accum=accum, lr=lr, grad=grad, indices=indices, momentum=momentum, use_locking=use_locking, use_nesterov=use_nesterov, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking"), "use_nesterov", _op._get_attr_bool("use_nesterov")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyMomentum", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyMomentum = tf_export("raw_ops.SparseApplyMomentum")(_ops.to_raw_op(sparse_apply_momentum)) def sparse_apply_momentum_eager_fallback(var, accum, lr, grad, indices, momentum, use_locking, use_nesterov, name, ctx): raise RuntimeError("sparse_apply_momentum op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_proximal_adagrad(var, accum, lr, l1, l2, grad, indices, use_locking=False, name=None): r"""Sparse update entries in '*var' and '*accum' according to FOBOS algorithm. That is for rows we have grad for, we update var and accum as follows: $$accum += grad * grad$$ $$prox_v = var$$ $$prox_v -= lr * grad * (1 / sqrt(accum))$$ $$var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). accum: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Learning rate. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If True, updating of the var and accum tensors will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_proximal_adagrad op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyProximalAdagrad", var=var, accum=accum, lr=lr, l1=l1, l2=l2, grad=grad, indices=indices, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyProximalAdagrad", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyProximalAdagrad = tf_export("raw_ops.SparseApplyProximalAdagrad")(_ops.to_raw_op(sparse_apply_proximal_adagrad)) def sparse_apply_proximal_adagrad_eager_fallback(var, accum, lr, l1, l2, grad, indices, use_locking, name, ctx): raise RuntimeError("sparse_apply_proximal_adagrad op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_proximal_gradient_descent(var, alpha, l1, l2, grad, indices, use_locking=False, name=None): r"""Sparse update '*var' as FOBOS algorithm with fixed learning rate. That is for rows we have grad for, we update var as follows: $$prox_v = var - alpha * grad$$ $$var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). alpha: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. l1: A `Tensor`. Must have the same type as `var`. L1 regularization. Must be a scalar. l2: A `Tensor`. Must have the same type as `var`. L2 regularization. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var and accum. use_locking: An optional `bool`. Defaults to `False`. If True, the subtraction will be protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_proximal_gradient_descent op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyProximalGradientDescent", var=var, alpha=alpha, l1=l1, l2=l2, grad=grad, indices=indices, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyProximalGradientDescent", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyProximalGradientDescent = tf_export("raw_ops.SparseApplyProximalGradientDescent")(_ops.to_raw_op(sparse_apply_proximal_gradient_descent)) def sparse_apply_proximal_gradient_descent_eager_fallback(var, alpha, l1, l2, grad, indices, use_locking, name, ctx): raise RuntimeError("sparse_apply_proximal_gradient_descent op does not support eager execution. Arg 'out' is a ref.") def sparse_apply_rms_prop(var, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking=False, name=None): r"""Update '*var' according to the RMSProp algorithm. Note that in dense implementation of this algorithm, ms and mom will update even if the grad is zero, but in this sparse implementation, ms and mom will not update in iterations during which the grad is zero. mean_square = decay * mean_square + (1-decay) * gradient ** 2 Delta = learning_rate * gradient / sqrt(mean_square + epsilon) $$ms <- rho * ms_{t-1} + (1-rho) * grad * grad$$ $$mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)$$ $$var <- var - mom$$ Args: var: A mutable `Tensor`. Must be one of the following types: `float32`, `float64`, `int32`, `uint8`, `int16`, `int8`, `complex64`, `int64`, `qint8`, `quint8`, `qint32`, `bfloat16`, `uint16`, `complex128`, `half`, `uint32`, `uint64`. Should be from a Variable(). ms: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). mom: A mutable `Tensor`. Must have the same type as `var`. Should be from a Variable(). lr: A `Tensor`. Must have the same type as `var`. Scaling factor. Must be a scalar. rho: A `Tensor`. Must have the same type as `var`. Decay rate. Must be a scalar. momentum: A `Tensor`. Must have the same type as `var`. epsilon: A `Tensor`. Must have the same type as `var`. Ridge term. Must be a scalar. grad: A `Tensor`. Must have the same type as `var`. The gradient. indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. A vector of indices into the first dimension of var, ms and mom. use_locking: An optional `bool`. Defaults to `False`. If `True`, updating of the var, ms, and mom tensors is protected by a lock; otherwise the behavior is undefined, but may exhibit less contention. name: A name for the operation (optional). Returns: A mutable `Tensor`. Has the same type as `var`. """ _ctx = _context._context or _context.context() tld = _ctx._thread_local_data if tld.is_eager: raise RuntimeError("sparse_apply_rms_prop op does not support eager execution. Arg 'out' is a ref.") # Add nodes to the TensorFlow graph. if use_locking is None: use_locking = False use_locking = _execute.make_bool(use_locking, "use_locking") _, _, _op, _outputs = _op_def_library._apply_op_helper( "SparseApplyRMSProp", var=var, ms=ms, mom=mom, lr=lr, rho=rho, momentum=momentum, epsilon=epsilon, grad=grad, indices=indices, use_locking=use_locking, name=name) _result = _outputs[:] if _execute.must_record_gradient(): _attrs = ("T", _op._get_attr_type("T"), "Tindices", _op._get_attr_type("Tindices"), "use_locking", _op._get_attr_bool("use_locking")) _inputs_flat = _op.inputs _execute.record_gradient( "SparseApplyRMSProp", _inputs_flat, _attrs, _result) _result, = _result return _result SparseApplyRMSProp = tf_export("raw_ops.SparseApplyRMSProp")(_ops.to_raw_op(sparse_apply_rms_prop)) def sparse_apply_rms_prop_eager_fallback(var, ms, mom, lr, rho, momentum, epsilon, grad, indices, use_locking, name, ctx): raise RuntimeError("sparse_apply_rms_prop op does not support eager execution. Arg 'out' is a ref.")