"""Save the list of files matching pattern, so it is only computed once.

with ops.name_scope(name, "matching_filenames", [pattern]) as name:

return vs.variable(

name=name, initial_value=io_ops.matching_files(pattern),

trainable=False, validate_shape=False,

"""Returns tensor `num_epochs` times and then raises an `OutOfRange` error.

if num_epochs is None:

return tensor

if num_epochs <= 0:

raise ValueError("num_epochs must be > 0 not %d." % num_epochs)

with ops.name_scope(name, "limit_epochs", [tensor]) as name:

zero64 = constant_op.constant(0, dtype=dtypes.int64)

epochs = vs.variable(

zero64, name="epochs", trainable=False,

collections=[ops.GraphKeys.LOCAL_VARIABLES])

counter = epochs.count_up_to(num_epochs)

with ops.control_dependencies([counter]):

element_shape=None,

num_epochs=None,

shuffle=True,

seed=None,

capacity=32,

shared_name=None,

summary_name=None,

name=None,

cancel_op=None):

"""Output the rows of `input_tensor` to a queue for an input pipeline.

with ops.name_scope(name, "input_producer", [input_tensor]):

input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")

element_shape = input_tensor.get_shape()[1:].merge_with(element_shape)

if not element_shape.is_fully_defined():

raise ValueError("Either `input_tensor` must have a fully defined shape "

"or `element_shape` must be specified")

if shuffle:

input_tensor = random_ops.random_shuffle(input_tensor, seed=seed)

input_tensor = limit_epochs(input_tensor, num_epochs)

q = data_flow_ops.FIFOQueue(capacity=capacity,

dtypes=[input_tensor.dtype.base_dtype],

shapes=[element_shape],

shared_name=shared_name, name=name)

enq = q.enqueue_many([input_tensor])

queue_runner.add_queue_runner(

queue_runner.QueueRunner(

q, [enq], cancel_op=cancel_op))

if summary_name is not None:

summary.scalar(summary_name,

math_ops.cast(q.size(), dtypes.float32) * (1. / capacity))

num_epochs=None,

shuffle=True,

seed=None,

capacity=32,

shared_name=None,

name=None,

cancel_op=None):

"""Output strings (e.g. filenames) to a queue for an input pipeline.

not_null_err = "string_input_producer requires a non-null input tensor"

if not isinstance(string_tensor, ops.Tensor) and not string_tensor:

raise ValueError(not_null_err)

with ops.name_scope(name, "input_producer", [string_tensor]) as name:

string_tensor = ops.convert_to_tensor(string_tensor, dtype=dtypes.string)

with ops.control_dependencies([

control_flow_ops.Assert(

math_ops.greater(array_ops.size(string_tensor), 0),

[not_null_err])]):

string_tensor = array_ops.identity(string_tensor)

return input_producer(

input_tensor=string_tensor,

element_shape=[],

num_epochs=num_epochs,

shuffle=shuffle,

seed=seed,

capacity=capacity,

shared_name=shared_name,

name=name,

summary_name="fraction_of_%d_full" % capacity,

capacity=32, shared_name=None, name=None):

"""Produces the integers from 0 to limit-1 in a queue.

with ops.name_scope(name, "input_producer", [limit]) as name:

range_tensor = math_ops.range(limit)

return input_producer(

range_tensor, [], num_epochs, shuffle, seed, capacity,

capacity=32, shared_name=None, name=None):

"""Produces a slice of each `Tensor` in `tensor_list`.

with ops.name_scope(name, "input_producer", tensor_list):

tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensor_list)

if not tensor_list:

raise ValueError(

"Expected at least one tensor in slice_input_producer().")

range_size = array_ops.shape(tensor_list[0])[0]

# TODO(josh11b): Add an assertion that the first dimension of

# everything in TensorList matches. Maybe just check the inferred shapes?

queue = range_input_producer(range_size, num_epochs=num_epochs,

shuffle=shuffle, seed=seed, capacity=capacity,

shared_name=shared_name)

index = queue.dequeue()

output = [array_ops.gather(t, index) for t in tensor_list]

"""Store information about the Tensor: Is it sparse?, map_op, and rank."""

def __init__(self, sparse, map_op, rank):

"""Create the metadata.

self._sparse = sparse

self._map_op = map_op

self._rank = rank

def __eq__(self, other):

if self.sparse != other.sparse:

return False

if not self.sparse:

return True

# If map_ops are not the same, the data source is not the same.

if (self.map_op is not None) != (other.map_op is not None):

return False

if self.map_op != other.map_op:

return False

if not self.rank.is_compatible_with(other.rank):

return False

return True

def __ne__(self, other):

return not self.__eq__(other)

def __str__(self):

return "[SparseMetaData(%s, %s, %s)]" % (self.sparse, self.map_op.name,

self.rank)

def merge_with(self, other):

if self != other:

raise ValueError("SparseMetaData objects are incompatible: %s vs. %s"

% (self, other))

if self.sparse:

self.rank.merge_with(other.rank)

return self

@property

def map_op(self):

return self._map_op

@property

def sparse(self):

return self._sparse

@property

def rank(self):

if isinstance(tensors, dict):

return [tensors[k] for k in sorted(tensors)]

else:

if not tensors_list:

raise ValueError("Expected at least one set of tensors")

if isinstance(tensors_list[0], dict):

expected_keys = set(tensors_list[0].keys())

for tensors in tensors_list[1:]:

if set(tensors.keys()) != expected_keys:

raise ValueError("All dictionaries in tensors_list must have "

"the same keys")

return [_as_tensor_list(tensors) for tensors in tensors_list]

else:

if isinstance(original_tensors, dict):

if len(original_tensors) == 1:

# tensor_list is bogusly returned as a single tensor if only one tensor

# was enqueued. Make it a list again. See b/28117485.

tensor_list = [tensor_list]

return {k: tensor_list[i]

for i, k in enumerate(sorted(original_tensors))}

else:

"""A `tf.cond` that does nothing when the condition is static."""

pred = ops.convert_to_tensor(pred)

static_pred = tensor_util.constant_value(pred)

if static_pred is not None:

if static_pred:

return if_true()

else:

return if_false()

else:

return control_flow_ops.cond(

pred,

if_true,

shared_map_ops=None):

"""Store SparseTensors for feeding into batch, etc.

maybe_shared_map_ops = shared_map_ops or [None] * len(tensor_list)

def _sparse_meta_data(t, storing_op, map_op):

if not isinstance(t, sparse_tensor.SparseTensor):

return _SparseMetaData(False, None, None)

rank = t.dense_shape.get_shape().with_rank(1)[0]

if enqueue_many:

rank -= 1

# If a shared map_op was provided, use that. Otherwise use the name of

# the operation used to store the SparseTensor.

return _SparseMetaData(

sparse=True, map_op=map_op or storing_op, rank=rank)

def _maybe_store(t, shared_map_op):

"""Store Sparse tensor, if necessary."""

if not isinstance(t, sparse_tensor.SparseTensor):

return t

map_op_name = shared_map_op.name if shared_map_op else None

def _maybe_store_sparse(t, map_op_name, keep_input):

"""Conditionally store a single sparse Tensor."""

return _smart_cond(

keep_input,

lambda: _store_sparse(t, shared_name=map_op_name),

lambda: constant_op.constant(-1, dtypes.int64))

def _maybe_store_many_sparse(t, map_op_name, keep_input):

"""Conditionally store multiple sparse Tensors."""

out_tensor = _smart_cond(

keep_input,

lambda: _store_many_sparse(t, shared_name=map_op_name),

lambda: -1 * array_ops.ones(array_ops.shape(t)[0:1], dtypes.int64))

out_tensor.set_shape([None]) # necessary when t.ndims is unknown

return out_tensor

if keep_input.get_shape().ndims == 1:

t = sparse_ops.sparse_retain(t, keep_input)

store_f = lambda t, name, _: _store_many_sparse(t, shared_name=name)

elif enqueue_many:

store_f = _maybe_store_many_sparse

else:

store_f = _maybe_store_sparse

return store_f(t, map_op_name, keep_input)

stored_list = [

_maybe_store(t, shared_map_op) for t, shared_map_op

in zip(tensor_list, maybe_shared_map_ops)]

# Since the output of `_store{_many}_sparse is wrapped in a tf.cond `Merge`,

# we can't just get the Op of the resulting tensor.

def _sparse_op(stored):

for input_tensor in stored.op.inputs:

if input_tensor.op.type in ("AddSparseToTensorsMap",

"AddManySparseToTensorsMap"):

return input_tensor.op

# If there was no sparse input, then the original stored Tensor wasn't

# sparse and we can just return the original Tensor's Op.

return stored.op

sparse_info_list = [

_sparse_meta_data(t, _sparse_op(stored), shared_map_op)

for t, stored, shared_map_op

in zip(tensor_list, stored_list, maybe_shared_map_ops)]

# Expand dims of stored tensors by 1 for proper enqueue shape

stored_list = [

array_ops.expand_dims(s, [-1]) if s_info.sparse else s

for s, s_info in zip(stored_list, sparse_info_list)]

"""Store SparseTensors for feeding into batch_join, etc."""

(s0, sparse_info_list) = _store_sparse_tensors(

tensor_list_list[0], enqueue_many, keep_input)

stored_list_list = [s0]

for tensor_list in tensor_list_list[1:]:

s, sparse_info_candidate = _store_sparse_tensors(

tensor_list, enqueue_many, keep_input,

[st.map_op for st in sparse_info_list])

if sparse_info_list != sparse_info_candidate:

raise ValueError("Inconsistent SparseTensors list: %s vs. %s"

% (tensor_list_list[0], tensor_list))

sparse_info_list = [

info.merge_with(candidate)

for (info, candidate) in zip(sparse_info_list, sparse_info_candidate)]

stored_list_list.append(s)

"""Restore SparseTensors after dequeue in batch, batch_join, etc."""

received_sequence = isinstance(stored_list, collections.Sequence)

if not received_sequence:

stored_list = (stored_list,)

tensors = [

_restore_sparse(sparse_map_op=info.map_op,

sparse_handles=array_ops.squeeze(s, [1]),

rank=(info.rank + 1).value)

if info.sparse else s

for (s, info) in zip(stored_list, sparse_info_list)]

tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensor_list)

if not tensor_list:

raise ValueError("Expected at least one tensor in batch().")

tensor_list_list = [ops.convert_n_to_tensor_or_indexed_slices(tl)

for tl in tensor_list_list]

if not tensor_list_list:

raise ValueError("Expected at least one input in batch_join().")

"""Validate `keep_input` argument to conditional batching functions."""

keep_input = ops.convert_to_tensor(keep_input)

if keep_input.get_shape().ndims is None:

raise ValueError(

"`keep_input` dimensions must be known at graph construction.")

if not enqueue_many and keep_input.get_shape().ndims == 1:

raise ValueError(

"`keep_input` cannot be a vector when `enqueue_many=False`.")

if keep_input.get_shape().ndims > 1:

raise ValueError("`keep_input` must be 0 or 1 dimensions.")

all_types = [[t.dtype for t in tl] for tl in tensor_list_list]

types = all_types[0]

for other_types in all_types[1:]:

if other_types != types:

raise TypeError("Expected types to be consistent: %s vs. %s." %

(", ".join(x.name for x in types),

", ".join(x.name for x in other_types)))

shape_list = [tensor_shape.as_shape(s) for s in shape_list]

if enqueue_many:

# We want the shapes without the leading batch dimension.

shape_list = [s.with_rank_at_least(1)[1:] for s in shape_list]

merged_shape = shape_list[0]

for s in shape_list[1:]:

merged_shape.merge_with(s)

"""Calculate and merge the shapes of incoming tensors.

if shapes is None:

len0 = len(tensor_list_list[0])

for tl in tensor_list_list:

for i in xrange(len0):

if tl[i].get_shape().ndims is None:

raise ValueError("Cannot infer Tensor's rank: %s" % tl[i])

shapes = [_merge_shapes(

[tl[i].get_shape().as_list() for tl in tensor_list_list], enqueue_many)

for i in xrange(len0)]

"""Select which examples to enqueue based on vector `keep_input`."""

select_i = math_ops.cast(keep_input, dtypes.int32)

tensor_list = [

data_flow_ops.dynamic_partition(x, select_i, num_partitions=2)[1]

for x in tensor_list]

"""Enqueue `tensor_list_list` in `queue`."""

if enqueue_many:

enqueue_fn = queue.enqueue_many

else:

enqueue_fn = queue.enqueue

if keep_input.get_shape().ndims == 1:

enqueue_ops = [enqueue_fn(_select_which_to_enqueue(x, keep_input))

for x in tensor_list_list]

else:

enqueue_ops = [_smart_cond(

keep_input,

lambda: enqueue_fn(tl), # pylint:disable=cell-var-from-loop

control_flow_ops.no_op) for tl in tensor_list_list]

"""Enqueue `tensor_list` in `queue`."""

if enqueue_many:

enqueue_fn = queue.enqueue_many

else:

enqueue_fn = queue.enqueue

if keep_input.get_shape().ndims == 1:

enqueue_ops = [

enqueue_fn(_select_which_to_enqueue(tensor_list, keep_input))] * threads

else:

enqueue_ops = [_smart_cond(

keep_input,

lambda: enqueue_fn(tensor_list),

control_flow_ops.no_op)] * threads

enqueue_many=False, shapes=None, dynamic_pad=False,

allow_smaller_final_batch=False, shared_name=None,

name=None):

"""Helper function for `batch` and `maybe_batch`."""

tensor_list = _as_tensor_list(tensors)

with ops.name_scope(name, "batch", list(tensor_list) + [keep_input]) as name:

tensor_list = _validate(tensor_list)

keep_input = _validate_keep_input(keep_input, enqueue_many)

(tensor_list, sparse_info) = _store_sparse_tensors(

tensor_list, enqueue_many, keep_input)

types = _dtypes([tensor_list])

shapes = _shapes([tensor_list], shapes, enqueue_many)

# TODO(josh11b,mrry): Switch to BatchQueue once it is written.

queue = _which_queue(dynamic_pad)(

capacity=capacity, dtypes=types, shapes=shapes, shared_name=shared_name)

_enqueue(queue, tensor_list, num_threads, enqueue_many, keep_input)

summary.scalar("fraction_of_%d_full" % capacity,

math_ops.cast(queue.size(), dtypes.float32) *

(1. / capacity))

if allow_smaller_final_batch:

dequeued = queue.dequeue_up_to(batch_size, name=name)

else:

dequeued = queue.dequeue_many(batch_size, name=name)

dequeued = _restore_sparse_tensors(dequeued, sparse_info)

enqueue_many=False, shapes=None, dynamic_pad=False,

allow_smaller_final_batch=False, shared_name=None, name=None):

"""Helper function for `batch_join` and `maybe_batch_join`."""

tensor_list_list = _as_tensor_list_list(tensors_list)

with ops.name_scope(name, "batch_join",

_flatten(tensor_list_list) + [keep_input]) as name:

tensor_list_list = _validate_join(tensor_list_list)

keep_input = _validate_keep_input(keep_input, enqueue_many)

tensor_list_list, sparse_info = _store_sparse_tensors_join(

tensor_list_list, enqueue_many, keep_input)

types = _dtypes(tensor_list_list)

shapes = _shapes(tensor_list_list, shapes, enqueue_many)

# TODO(josh11b,mrry): Switch to BatchQueue once it is written.

queue = _which_queue(dynamic_pad)(

capacity=capacity, dtypes=types, shapes=shapes, shared_name=shared_name)

_enqueue_join(queue, tensor_list_list, enqueue_many, keep_input)

summary.scalar("fraction_of_%d_full" % capacity,

math_ops.cast(queue.size(), dtypes.float32) *

(1. / capacity))

if allow_smaller_final_batch:

dequeued = queue.dequeue_up_to(batch_size, name=name)

else:

dequeued = queue.dequeue_many(batch_size, name=name)

dequeued = _restore_sparse_tensors(dequeued, sparse_info)

# tensors_list was validated to not be empty.

keep_input, num_threads=1, seed=None, enqueue_many=False,

shapes=None, allow_smaller_final_batch=False,

shared_name=None, name=None):

"""Helper function for `shuffle_batch` and `maybe_shuffle_batch`."""

tensor_list = _as_tensor_list(tensors)

with ops.name_scope(name, "shuffle_batch",

list(tensor_list) + [keep_input]) as name:

tensor_list = _validate(tensor_list)

keep_input = _validate_keep_input(keep_input, enqueue_many)

tensor_list, sparse_info = _store_sparse_tensors(

tensor_list, enqueue_many, keep_input)

types = _dtypes([tensor_list])

shapes = _shapes([tensor_list], shapes, enqueue_many)

queue = data_flow_ops.RandomShuffleQueue(

capacity=capacity, min_after_dequeue=min_after_dequeue, seed=seed,

dtypes=types, shapes=shapes, shared_name=shared_name)

_enqueue(queue, tensor_list, num_threads, enqueue_many, keep_input)

full = (math_ops.cast(math_ops.maximum(0, queue.size() - min_after_dequeue),

dtypes.float32) *

(1. / (capacity - min_after_dequeue)))

# Note that name contains a '/' at the end so we intentionally do not place

# a '/' after %s below.

summary_name = (

"fraction_over_%d_of_%d_full" %

(min_after_dequeue, capacity - min_after_dequeue))

summary.scalar(summary_name, full)

if allow_smaller_final_batch:

dequeued = queue.dequeue_up_to(batch_size, name=name)

else:

dequeued = queue.dequeue_many(batch_size, name=name)

dequeued = _restore_sparse_tensors(dequeued, sparse_info)

min_after_dequeue, keep_input, seed=None,

enqueue_many=False, shapes=None,

allow_smaller_final_batch=False, shared_name=None,

name=None):

"""Helper function for `shuffle_batch_join` and `maybe_shuffle_batch_join`."""

tensor_list_list = _as_tensor_list_list(tensors_list)

with ops.name_scope(name, "shuffle_batch_join",

_flatten(tensor_list_list) + [keep_input]) as name:

tensor_list_list = _validate_join(tensor_list_list)

keep_input = _validate_keep_input(keep_input, enqueue_many)

tensor_list_list, sparse_info = _store_sparse_tensors_join(

tensor_list_list, enqueue_many, keep_input)

types = _dtypes(tensor_list_list)

shapes = _shapes(tensor_list_list, shapes, enqueue_many)

queue = data_flow_ops.RandomShuffleQueue(

capacity=capacity, min_after_dequeue=min_after_dequeue, seed=seed,

dtypes=types, shapes=shapes, shared_name=shared_name)

_enqueue_join(queue, tensor_list_list, enqueue_many, keep_input)

full = (math_ops.cast(math_ops.maximum(0, queue.size() - min_after_dequeue),

dtypes.float32) *

(1. / (capacity - min_after_dequeue)))

# Note that name contains a '/' at the end so we intentionally do not place

# a '/' after %s below.

summary_name = (

"fraction_over_%d_of_%d_full" %

(min_after_dequeue, capacity - min_after_dequeue))

summary.scalar(summary_name, full)

if allow_smaller_final_batch:

dequeued = queue.dequeue_up_to(batch_size, name=name)

else:

dequeued = queue.dequeue_many(batch_size, name=name)

dequeued = _restore_sparse_tensors(dequeued, sparse_info)

# tensors_list was validated to not be empty.

enqueue_many=False, shapes=None, dynamic_pad=False,

allow_smaller_final_batch=False, shared_name=None, name=None):

"""Creates batches of tensors in `tensors`.

return _batch(

tensors,

batch_size,

keep_input=True,

num_threads=num_threads,

capacity=capacity,

enqueue_many=enqueue_many,

shapes=shapes,

dynamic_pad=dynamic_pad,

allow_smaller_final_batch=allow_smaller_final_batch,

shared_name=shared_name,

enqueue_many=False, shapes=None, dynamic_pad=False,

allow_smaller_final_batch=False, shared_name=None, name=None):

"""Conditionally creates batches of tensors based on `keep_input`.

return _batch(

tensors,

batch_size,

keep_input,

num_threads=num_threads,

capacity=capacity,

enqueue_many=enqueue_many,

shapes=shapes,

dynamic_pad=dynamic_pad,

allow_smaller_final_batch=allow_smaller_final_batch,

shared_name=shared_name,

shapes=None, dynamic_pad=False, allow_smaller_final_batch=False,

shared_name=None, name=None):

"""Runs a list of tensors to fill a queue to create batches of examples.