"""Computes log Poisson loss given `log_input`.

with ops.name_scope(name, "log_poisson_loss", [log_input, targets]) as name:

log_input = ops.convert_to_tensor(log_input, name="log_input")

targets = ops.convert_to_tensor(targets, name="targets")

try:

targets.get_shape().merge_with(log_input.get_shape())

except ValueError:

raise ValueError(

"log_input and targets must have the same shape (%s vs %s)" %

(log_input.get_shape(), targets.get_shape()))

result = math_ops.exp(log_input) - log_input * targets

if compute_full_loss:

# need to create constant tensors here so that their dtypes can be matched

# to that of the targets.

point_five = constant_op.constant(0.5, dtype=targets.dtype)

two_pi = constant_op.constant(2 * math.pi, dtype=targets.dtype)

stirling_approx = (targets * math_ops.log(targets)) - targets + (

point_five * math_ops.log(two_pi * targets))

zeros = array_ops.zeros_like(targets, dtype=targets.dtype)

ones = array_ops.ones_like(targets, dtype=targets.dtype)

cond = math_ops.logical_and(targets >= zeros, targets <= ones)

result += array_ops.where(cond, zeros, stirling_approx)

_sentinel=None,

labels=None,

logits=None,

name=None):

"""Computes sigmoid cross entropy given `logits`.

# pylint: disable=protected-access

nn_ops._ensure_xent_args("sigmoid_cross_entropy_with_logits", _sentinel,

labels, logits)

# pylint: enable=protected-access

with ops.name_scope(name, "logistic_loss", [logits, labels]) as name:

logits = ops.convert_to_tensor(logits, name="logits")

labels = ops.convert_to_tensor(labels, name="labels")

try:

labels.get_shape().merge_with(logits.get_shape())

except ValueError:

raise ValueError("logits and labels must have the same shape (%s vs %s)" %

(logits.get_shape(), labels.get_shape()))

# The logistic loss formula from above is

# x - x * z + log(1 + exp(-x))

# For x < 0, a more numerically stable formula is

# -x * z + log(1 + exp(x))

# Note that these two expressions can be combined into the following:

# max(x, 0) - x * z + log(1 + exp(-abs(x)))

# To allow computing gradients at zero, we define custom versions of max and

# abs functions.

zeros = array_ops.zeros_like(logits, dtype=logits.dtype)

cond = (logits >= zeros)

relu_logits = array_ops.where(cond, logits, zeros)

neg_abs_logits = array_ops.where(cond, -logits, logits)

return math_ops.add(

relu_logits - logits * labels,

math_ops.log1p(math_ops.exp(neg_abs_logits)),

"""Computes a weighted cross entropy.

with ops.name_scope(name, "logistic_loss", [logits, targets]) as name:

logits = ops.convert_to_tensor(logits, name="logits")

targets = ops.convert_to_tensor(targets, name="targets")

try:

targets.get_shape().merge_with(logits.get_shape())

except ValueError:

raise ValueError(

"logits and targets must have the same shape (%s vs %s)" %

(logits.get_shape(), targets.get_shape()))

# The logistic loss formula from above is

# (1 - z) * x + (1 + (q - 1) * z) * log(1 + exp(-x))

# For x < 0, a more numerically stable formula is

# (1 - z) * x + (1 + (q - 1) * z) * log(1 + exp(x)) - l * x

# To avoid branching, we use the combined version

# (1 - z) * x + l * (log(1 + exp(-abs(x))) + max(-x, 0))

log_weight = 1 + (pos_weight - 1) * targets

return math_ops.add(

(1 - targets) * logits,

log_weight * (math_ops.log1p(math_ops.exp(-math_ops.abs(logits))) +

nn_ops.relu(-logits)),

"""Computes Relu(x * weight + biases).

with ops.name_scope(name, "relu_layer", [x, weights, biases]) as name:

x = ops.convert_to_tensor(x, name="x")

weights = ops.convert_to_tensor(weights, name="weights")

biases = ops.convert_to_tensor(biases, name="biases")

xw_plus_b = nn_ops.bias_add(math_ops.matmul(x, weights), biases)

"""Normalizes along dimension `dim` using an L2 norm.

with ops.name_scope(name, "l2_normalize", [x]) as name:

x = ops.convert_to_tensor(x, name="x")

square_sum = math_ops.reduce_sum(math_ops.square(x), dim, keep_dims=True)

x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon))

"""Returns the fraction of zeros in `value`.

with ops.name_scope(name, "zero_fraction", [value]):

value = ops.convert_to_tensor(value, name="value")

zero = constant_op.constant(0, dtype=value.dtype, name="zero")

return math_ops.reduce_mean(

filter,

strides,

padding,

rate=None,

name=None,

data_format=None):

"""Depthwise 2-D convolution.

with ops.name_scope(name, "depthwise", [input, filter]) as name:

input = ops.convert_to_tensor(input, name="tensor_in")

filter = ops.convert_to_tensor(filter, name="filter_in")

if rate is None:

rate = [1, 1]

def op(input_converted, _, padding):

return nn_ops.depthwise_conv2d_native(

input=input_converted,

filter=filter,

strides=strides,

padding=padding,

data_format=data_format,

name=name)

return nn_ops.with_space_to_batch(

input=input,

filter_shape=array_ops.shape(filter),

dilation_rate=rate,

padding=padding,

data_format=data_format,

depthwise_filter,

pointwise_filter,

strides,

padding,

rate=None,

name=None,

data_format=None):

"""2-D convolution with separable filters.

with ops.name_scope(name, "separable_conv2d",

[input, depthwise_filter, pointwise_filter]) as name:

input = ops.convert_to_tensor(input, name="tensor_in")

depthwise_filter = ops.convert_to_tensor(

depthwise_filter, name="depthwise_filter")

pointwise_filter = ops.convert_to_tensor(

pointwise_filter, name="pointwise_filter")

pointwise_filter_shape = pointwise_filter.get_shape().with_rank(4)

pointwise_filter_shape[0].assert_is_compatible_with(1)

pointwise_filter_shape[1].assert_is_compatible_with(1)

if rate is None:

rate = [1, 1]

# The layout of the ops in the graph are expected to be as follows:

# depthwise_conv2d // Conv2D op corresponding to native deptwise conv.

# separable_conv2d // Conv2D op corresponding to the pointwise conv.

def op(input_converted, _, padding):

return nn_ops.depthwise_conv2d_native(

input=input_converted,

filter=depthwise_filter,

strides=strides,

padding=padding,

data_format=data_format,

name="depthwise")

depthwise = nn_ops.with_space_to_batch(

input=input,

filter_shape=array_ops.shape(depthwise_filter),

dilation_rate=rate,

padding=padding,

data_format=data_format,

op=op)

return nn_ops.conv2d(

depthwise,

pointwise_filter, [1, 1, 1, 1],

padding="VALID",

data_format=data_format,

"""Calculate the sufficient statistics for the mean and variance of `x`.

axes = list(set(axes))

with ops.name_scope(name, "sufficient_statistics", [x, shift]):

x = ops.convert_to_tensor(x, name="x")

x_shape = x.get_shape()

if all(x_shape[d].value is not None for d in axes):

counts = 1

for d in axes:

counts *= x_shape[d].value

counts = constant_op.constant(counts, dtype=x.dtype)

else: # shape needs to be inferred at runtime.

x_dims = array_ops.gather(

math_ops.cast(array_ops.shape(x), x.dtype), axes)

counts = math_ops.reduce_prod(x_dims, name="count")

if shift is not None:

shift = ops.convert_to_tensor(shift, name="shift")

m_ss = math_ops.subtract(x, shift)

v_ss = math_ops.squared_difference(x, shift)

else: # no shift.

m_ss = x

v_ss = math_ops.square(x)

m_ss = math_ops.reduce_sum(m_ss, axes, keep_dims=keep_dims, name="mean_ss")

v_ss = math_ops.reduce_sum(v_ss, axes, keep_dims=keep_dims, name="var_ss")

"""Calculate the mean and variance of based on the sufficient statistics.

with ops.name_scope(name, "normalize", [counts, mean_ss, variance_ss, shift]):

divisor = math_ops.reciprocal(counts, name="divisor")

if shift is not None:

shifted_mean = math_ops.multiply(mean_ss, divisor, name="shifted_mean")

mean = math_ops.add(shifted_mean, shift, name="mean")

else: # no shift.

shifted_mean = math_ops.multiply(mean_ss, divisor, name="mean")

mean = shifted_mean

variance = math_ops.subtract(

math_ops.multiply(variance_ss, divisor),

math_ops.square(shifted_mean),

name="variance")

shift=None, # pylint: disable=unused-argument

name=None, keep_dims=False):

"""Calculate the mean and variance of `x`.

with ops.name_scope(name, "moments", [x, axes]):

# The dynamic range of fp16 is too limited to support the collection of

# sufficient statistics. As a workaround we simply perform the operations

# on 32-bit floats before converting the mean and variance back to fp16

y = math_ops.cast(x, dtypes.float32) if x.dtype == dtypes.float16 else x

# Compute true mean while keeping the dims for proper broadcasting.

mean = math_ops.reduce_mean(y, axes, keep_dims=True, name="mean")

# sample variance, not unbiased variance

variance = math_ops.reduce_mean(

math_ops.squared_difference(y, array_ops.stop_gradient(mean)),

axes,

keep_dims=True,

name="variance")

if not keep_dims:

mean = array_ops.squeeze(mean, axes)

variance = array_ops.squeeze(variance, axes)

if x.dtype == dtypes.float16:

return (math_ops.cast(mean, dtypes.float16), math_ops.cast(

variance, dtypes.float16))

else:

"""Returns the frequency-weighted mean and variance of `x`.

with ops.name_scope(name, "weighted_moments", [x, frequency_weights, axes]):

x = ops.convert_to_tensor(x, name="x")

frequency_weights = ops.convert_to_tensor(

frequency_weights, name="frequency_weights")

# Unlike moments(), this just uses a simpler two-pass method.

# See comment in moments() WRT precision; it applies here too.

needs_cast = x.dtype == dtypes.float16

if needs_cast:

x = math_ops.cast(x, dtypes.float32)

if frequency_weights.dtype != x.dtype:

frequency_weights = math_ops.cast(frequency_weights, x.dtype)

# Note that we use keep_dims=True for our reductions regardless of the arg;

# this is so that the results remain broadcast-compatible with the inputs.

weighted_input_sum = math_ops.reduce_sum(

frequency_weights * x, axes, name="weighted_input_sum", keep_dims=True)

# The shape of the weights isn't necessarily the same as x's

# shape, just broadcast-compatible with it -- so this expression

# performs broadcasting to give a per-item weight, with the same

# shape as (freqency_weights * x). This avoids having to reason

# through all the broadcast logic to compute a correct

# sum_of_weights.

broadcasted_weights = frequency_weights + array_ops.zeros_like(x)

sum_of_weights = math_ops.reduce_sum(

broadcasted_weights, axes, name="sum_of_weights", keep_dims=True)

divisor = math_ops.reciprocal(sum_of_weights, name="inv_weight_sum")

weighted_mean = math_ops.multiply(weighted_input_sum, divisor)

# Have the weighted mean; now on to variance:

weighted_distsq = math_ops.reduce_sum(

frequency_weights * math_ops.squared_difference(x, weighted_mean),

axes,

name="weighted_distsq",

keep_dims=True)

weighted_variance = math_ops.multiply(weighted_distsq, divisor)

if not keep_dims:

weighted_mean = array_ops.squeeze(weighted_mean, squeeze_dims=axes)

weighted_variance = array_ops.squeeze(

weighted_variance, squeeze_dims=axes)

if needs_cast:

weighted_mean = math_ops.cast(weighted_mean, dtypes.float16)

weighted_variance = math_ops.cast(weighted_variance, dtypes.float16)

mean,

variance,

offset,

scale,

variance_epsilon,

name=None):

r"""Batch normalization.

with ops.name_scope(name, "batchnorm", [x, mean, variance, scale, offset]):

inv = math_ops.rsqrt(variance + variance_epsilon)

if scale is not None:

inv *= scale

return x * inv + (offset - mean * inv

x,

scale,

offset, # pylint: disable=invalid-name

mean=None,

variance=None,

epsilon=0.001,

data_format="NHWC",

is_training=True,

name=None):

r"""Batch normalization.

x = ops.convert_to_tensor(x, name="input")

scale = ops.convert_to_tensor(scale, name="scale")

offset = ops.convert_to_tensor(offset, name="offset")

if is_training:

if (mean is not None) or (variance is not None):

raise ValueError("Both 'mean' and 'variance' must be None "

"if is_training is True.")

if mean is None:

mean = constant_op.constant([])

if variance is None:

variance = constant_op.constant([])

# Set a minimum epsilon to 1.001e-5, which is a requirement by CUDNN to

# prevent exception (see cudnn.h).

min_epsilon = 1.001e-5

epsilon = epsilon if epsilon > min_epsilon else min_epsilon

# TODO(reedwm): In a few weeks, switch to using the V2 version exclusively. We

# currently only use the V2 version for float16 inputs, which is not supported

# by the V1 version.

# pylint: disable=protected-access

if x.dtype == dtypes.float16:

fused_batch_norm_func = gen_nn_ops._fused_batch_norm_v2

else:

fused_batch_norm_func = gen_nn_ops._fused_batch_norm

# pylint: enable=protected-access

y, batch_mean, batch_var, _, _ = fused_batch_norm_func(

x,

scale,

offset,

mean,

variance,

epsilon=epsilon,

data_format=data_format,

is_training=is_training,

name=name)

m,

v,

beta,

gamma,

variance_epsilon,

scale_after_normalization,

name=None):

"""Batch normalization.

return batch_normalization(t, m, v, beta, gamma if scale_after_normalization

"""Returns a vector summing up each row of the matrix x."""

# _sum_rows(x) is equivalent to math_ops.reduce_sum(x, 1) when x is

# a matrix. The gradient of _sum_rows(x) is more efficient than

# reduce_sum(x, 1)'s gradient in today's implementation. Therefore,

# we use _sum_rows(x) in the nce_loss() computation since the loss

# is mostly used for training.

cols = array_ops.shape(x)[1]

ones_shape = array_ops.stack([cols, 1])

ones = array_ops.ones(ones_shape, x.dtype)

biases,

labels,

inputs,

num_sampled,

num_classes,

num_true=1,

sampled_values=None,

subtract_log_q=True,

remove_accidental_hits=False,

partition_strategy="mod",

name=None):

"""Helper function for nce_loss and sampled_softmax_loss functions.

if isinstance(weights, variables.PartitionedVariable):

weights = list(weights)

if not isinstance(weights, list):

weights = [weights]

with ops.name_scope(name, "compute_sampled_logits",

weights + [biases, inputs, labels]):

if labels.dtype != dtypes.int64:

labels = math_ops.cast(labels, dtypes.int64)

labels_flat = array_ops.reshape(labels, [-1])

# Sample the negative labels.

# sampled shape: [num_sampled] tensor

# true_expected_count shape = [batch_size, 1] tensor

# sampled_expected_count shape = [num_sampled] tensor

if sampled_values is None:

sampled_values = candidate_sampling_ops.log_uniform_candidate_sampler(

true_classes=labels,

num_true=num_true,

num_sampled=num_sampled,

unique=True,

range_max=num_classes)

# NOTE: pylint cannot tell that 'sampled_values' is a sequence

# pylint: disable=unpacking-non-sequence

sampled, true_expected_count, sampled_expected_count = (

array_ops.stop_gradient(s) for s in sampled_values)

# pylint: enable=unpacking-non-sequence

sampled = math_ops.cast(sampled, dtypes.int64)

# labels_flat is a [batch_size * num_true] tensor

# sampled is a [num_sampled] int tensor

all_ids = array_ops.concat([labels_flat, sampled], 0)

# Retrieve the true weights and the logits of the sampled weights.

# weights shape is [num_classes, dim]

all_w = embedding_ops.embedding_lookup(

weights, all_ids, partition_strategy=partition_strategy)

# true_w shape is [batch_size * num_true, dim]

true_w = array_ops.slice(all_w, [0, 0],

array_ops.stack(

[array_ops.shape(labels_flat)[0], -1]))

sampled_w = array_ops.slice(

all_w, array_ops.stack([array_ops.shape(labels_flat)[0], 0]), [-1, -1])

# inputs has shape [batch_size, dim]

# sampled_w has shape [num_sampled, dim]

# Apply X*W', which yields [batch_size, num_sampled]

sampled_logits = math_ops.matmul(inputs, sampled_w, transpose_b=True)

# Retrieve the true and sampled biases, compute the true logits, and

# add the biases to the true and sampled logits.

all_b = embedding_ops.embedding_lookup(

biases, all_ids, partition_strategy=partition_strategy)

# true_b is a [batch_size * num_true] tensor

# sampled_b is a [num_sampled] float tensor

true_b = array_ops.slice(all_b, [0], array_ops.shape(labels_flat))

sampled_b = array_ops.slice(all_b, array_ops.shape(labels_flat), [-1])