"""Load a tensor from an event file.

event = event_pb2.Event()

with gfile.Open(event_file_path, "rb") as f:

event.ParseFromString(f.read())

if (event.summary.value[0].tensor.tensor_content or

event.summary.value[0].tensor.string_val):

# Initialized tensor.

tensor_value = tensor_util.MakeNdarray(event.summary.value[0].tensor)

else:

# Uninitialized tensor.

tensor_value = None

event = event_pb2.Event()

with gfile.Open(event_file_path, "rb") as f:

event.ParseFromString(f.read())

"""Get the node name from a string that can be node or tensor name.

if ":" in name and not name.endswith(":"):

node_name = name[:name.rfind(":")]

output_slot = int(name[name.rfind(":") + 1:])

return node_name, output_slot

else:

"""Get the output slot number from the name of a graph element.

"""Get tensor name given node name and output slot index.

"""Get the string representation of a debug watch on a tensor.

"""Determine whether a node name is that of a debug Copy node.

"""Determine whether a node name is that of a debug node.

"""Parse the name of a debug node.

prefix = "__dbg_"

name = node_name

if not name.startswith(prefix):

raise ValueError("Invalid prefix in debug node name: '%s'" % node_name)

name = name[len(prefix):]

if name.count("_") < 2:

raise ValueError("Invalid debug node name: '%s'" % node_name)

debug_op = name[name.rindex("_") + 1:]

name = name[:name.rindex("_")]

debug_op_index = int(name[name.rindex("_") + 1:])

name = name[:name.rindex("_")]

if name.count(":") != 1:

raise ValueError("Invalid tensor name in debug node name: '%s'" % node_name)

watched_node_name = name[:name.index(":")]

watched_output_slot = int(name[name.index(":") + 1:])

"""A predicate for whether a tensor consists of any bad numerical values.

_ = datum # Datum metadata is unused in this predicate.

if tensor is None:

# Uninitialized tensor doesn't have bad numerical values.

return False

elif (np.issubdtype(tensor.dtype, np.float) or

np.issubdtype(tensor.dtype, np.complex) or

np.issubdtype(tensor.dtype, np.integer)):

return np.any(np.isnan(tensor)) or np.any(np.isinf(tensor))

else:

"""A single tensor dumped by TensorFlow Debugger (tfdbg).

def __init__(self, dump_root, debug_dump_rel_path):

"""`DebugTensorDatum` constructor.

base = os.path.basename(debug_dump_rel_path)

if base.count("_") < 3:

raise ValueError(

"Dump file path does not conform to the naming pattern: %s" % base)

# TODO(cais): Add hostname and pid to support dumps from distributed

# sessions.

self._timestamp = int(base.split("_")[-1])

self._debug_op = base.split("_")[-2]

self._output_slot = int(base.split("_")[-3])

namespace = os.path.dirname(debug_dump_rel_path)

node_base_name = "_".join(base.split("_")[:-3])

if not namespace or namespace == ".":

self._node_name = node_base_name

else:

self._node_name = namespace + "/" + node_base_name

self._file_path = os.path.join(dump_root, debug_dump_rel_path)

def __str__(self):

return "{DebugTensorDatum: %s:%d @ %s @ %d}" % (self.node_name,

self.output_slot,

self.debug_op,

self.timestamp)

def __repr__(self):

return self.__str__()

def get_tensor(self):

"""Get tensor from the dump (`Event`) file.

return load_tensor_from_event_file(self.file_path)

# TODO(cais): Add time unit suffix to timestamp and t0 (us).

@property

def timestamp(self):

"""Timestamp of when this tensor value was dumped.

return self._timestamp

@property

def debug_op(self):

"""Name of the debug op.

return self._debug_op

@property

def node_name(self):

"""Name of the node from which the tensor value was dumped.

return self._node_name

@property

def output_slot(self):

"""Output slot index from which the tensor value was dumped.

return self._output_slot

@property

def tensor_name(self):

"""Name of the tensor watched by the debug op.

return _get_tensor_name(self.node_name, self.output_slot)

@property

def watch_key(self):

"""Watch key identities a debug watch on a tensor.

return _get_tensor_watch_key(self.node_name, self.output_slot,

self.debug_op)

@property

def file_path(self):

"""Path to the file which stores the value of the dumped tensor."""

"""Data set from a debug-dump directory on filesystem.

def __init__(self, dump_root, partition_graphs=None, validate=True):

"""`DebugDumpDir` constructor.

if not gfile.IsDirectory(dump_root):

raise IOError("Dump root directory %s does not exist" % dump_root)

self._load_dumps(dump_root)

self._create_tensor_watch_maps()

self._load_partition_graphs(partition_graphs, validate)

self._python_graph = None

def _load_dumps(self, dump_root):

"""Load `DebugTensorDatum` instances from the dump root.

self._dump_root = dump_root

self._dump_tensor_data = []

self._dump_graph_file_paths = []

self._debug_watches = collections.defaultdict(

lambda: collections.defaultdict(set))

for root, _, files in gfile.Walk(self._dump_root):

for f in files:

if _is_graph_file(f):

self._dump_graph_file_paths.append(

os.path.join(self._dump_root, root, f))

continue

datum = self._dump_file_name_to_datum(root, f)

self._dump_tensor_data.append(datum)

self._debug_watches[datum.node_name][datum.output_slot].add(

datum.debug_op)

self._dump_tensor_data = sorted(

self._dump_tensor_data, key=lambda x: x.timestamp)

if self._dump_tensor_data:

self._t0 = self._dump_tensor_data[0].timestamp

else:

self._t0 = None

def _dump_file_name_to_datum(self, dir_name, file_name):

"""Obtain a DebugTensorDatum from the directory and file name.

# Calculate the relative path of the dump file with respect to the root.

debug_dump_rel_path = os.path.join(

os.path.relpath(dir_name, self._dump_root), file_name)

return DebugTensorDatum(self._dump_root, debug_dump_rel_path)

def _create_tensor_watch_maps(self):

"""Create maps from tensor watch keys to datum and to timestamps.

self._watch_key_to_datum = {}

self._watch_key_to_rel_time = {}

for datum in self._dump_tensor_data:

if datum.watch_key not in self._watch_key_to_datum:

self._watch_key_to_datum[datum.watch_key] = [datum]

self._watch_key_to_rel_time[datum.watch_key] = [

datum.timestamp - self._t0

]

else:

self._watch_key_to_datum[datum.watch_key].append(datum)

self._watch_key_to_rel_time[datum.watch_key].append(datum.timestamp -

self._t0)

def set_python_graph(self, python_graph):

"""Provide Python `Graph` object to the wrapper.

self._python_graph = python_graph

self._node_traceback = {}

if self._python_graph:

for op in self._python_graph.get_operations():

self._node_traceback[op.name] = op.traceback

@property

def dumped_tensor_data(self):

return self._dump_tensor_data

@property

def t0(self):

"""Absolute timestamp of the first dumped tensor.

return self._t0

@property

def size(self):

"""Total number of dumped tensors in the dump root directory.

return len(self._dump_tensor_data)

def _load_partition_graphs(self, partition_graphs, validate):

"""Load and process partition graphs.

if partition_graphs:

self._partition_graphs = partition_graphs

elif self._dump_graph_file_paths:

# In case partition graphs are not available from arguments, load them

# from the dump directory.

self._partition_graphs = [

_load_graph_def_from_event_file(dump_file_path)

for dump_file_path in self._dump_graph_file_paths

]

else:

self._partition_graphs = None

return

self._node_attributes = {}

self._node_inputs = {}

self._node_ctrl_inputs = {}

self._node_recipients = {}

self._node_ctrl_recipients = {}

self._devices = []

self._node_devices = {}

self._node_op_types = {}

self._copy_send_nodes = []

for pg in self._partition_graphs:

for node in pg.node:

self._process_partition_graph_node(node)

self._prune_non_control_edges_of_debug_ops()

self._prune_control_edges_of_debug_ops()

self._populate_recipient_maps()

if validate:

self._validate_dump_with_graphs()

def _process_partition_graph_node(self, node):

"""Process a node from the partition graphs.

if _is_debug_node(node.name):

# This is a debug node. Parse the node name and retrieve the

# information about debug watches on tensors. But do not include

# the node in the graph.

(watched_node_name, watched_output_slot, _,

debug_op) = _parse_debug_node_name(node.name)

self._debug_watches[watched_node_name][watched_output_slot].add(

debug_op)

return

if node.name in self._node_inputs:

raise ValueError("Duplicate node name: '%s'" % node.name)

self._node_attributes[node.name] = node.attr

if node.device not in self._devices and node.device:

self._devices.append(node.device)

self._node_inputs[node.name] = []

self._node_ctrl_inputs[node.name] = []

self._node_recipients[node.name] = []

self._node_ctrl_recipients[node.name] = []

self._node_devices[node.name] = node.device

self._node_op_types[node.name] = node.op

for inp in node.input:

if _is_copy_node(inp) and node.op == "_Send":

self._copy_send_nodes.append(node.name)

if inp.startswith("^"):

cinp = inp[1:]

self._node_ctrl_inputs[node.name].append(cinp)

else:

self._node_inputs[node.name].append(inp)

def _prune_nodes_from_input_and_recipient_maps(self, nodes_to_prune):

"""Prune nodes out of input and recipient maps.

for node in nodes_to_prune:

del self._node_inputs[node]

del self._node_ctrl_inputs[node]

del self._node_recipients[node]

del self._node_ctrl_recipients[node]

def _prune_non_control_edges_of_debug_ops(self):

"""Prune (non-control) edges related to debug ops.

copy_nodes = []

for node in self._node_inputs:

if node in self._copy_send_nodes:

continue

if _is_copy_node(node):

copy_nodes.append(node)

inputs = self._node_inputs[node]

for i in xrange(len(inputs)):

inp = inputs[i]

if _is_copy_node(inp):

# Find the input to the Copy node, which should be the original

# input to the node.

orig_inp = self._node_inputs[inp][0]

inputs[i] = orig_inp

self._prune_nodes_from_input_and_recipient_maps(copy_nodes)

self._prune_nodes_from_input_and_recipient_maps(self._copy_send_nodes)

def _prune_control_edges_of_debug_ops(self):

"""Prune control edges related to the debug ops."""

for node in self._node_ctrl_inputs:

ctrl_inputs = self._node_ctrl_inputs[node]

debug_op_inputs = []

for ctrl_inp in ctrl_inputs:

if _is_debug_node(ctrl_inp):

debug_op_inputs.append(ctrl_inp)

for debug_op_inp in debug_op_inputs:

ctrl_inputs.remove(debug_op_inp)

def _populate_recipient_maps(self):

"""Populate the map from node name to recipient(s) of its output(s)."""

for node in self._node_inputs:

inputs = self._node_inputs[node]

for inp in inputs:

inp = get_node_name(inp)

if inp not in self._node_recipients:

self._node_recipients[inp] = []

self._node_recipients[inp].append(node)

for node in self._node_ctrl_inputs:

ctrl_inputs = self._node_ctrl_inputs[node]

for ctrl_inp in ctrl_inputs:

if ctrl_inp in self._copy_send_nodes:

continue

if ctrl_inp not in self._node_ctrl_recipients:

self._node_ctrl_recipients[ctrl_inp] = []

self._node_ctrl_recipients[ctrl_inp].append(node)

def _validate_dump_with_graphs(self):

"""Validate the dumped tensor data against the partition graphs.

if not self._partition_graphs:

raise LookupError("No partition graphs loaded.")

# Verify that the node names in the dump data are all present in the

# partition graphs.

for datum in self._dump_tensor_data:

if datum.node_name not in self._node_inputs:

raise ValueError("Node name '%s' is not found in partition graphs." %

datum.node_name)

pending_inputs = {}

for node in self._node_inputs:

pending_inputs[node] = []

inputs = self._node_inputs[node]

for inp in inputs:

inp_node = get_node_name(inp)

inp_output_slot = get_output_slot(inp)

if (inp_node in self._debug_watches and

inp_output_slot in self._debug_watches[inp_node] and

(inp_node, inp_output_slot) not in pending_inputs[node]):

pending_inputs[node].append((inp_node, inp_output_slot))

for datum in self._dump_tensor_data:

node = datum.node_name

slot = datum.output_slot

if pending_inputs[node]:

raise ValueError("Causality violated in timing relations of debug "

"dumps: %s (%d): "

"these input(s) are not satisfied: %s" %

(node, datum.timestamp, repr(pending_inputs[node])))

recipients = self._node_recipients[node]

for recipient in recipients:

recipient_pending_inputs = pending_inputs[recipient]

if (node, slot) in recipient_pending_inputs:

if self.node_op_type(recipient) == "Merge":

# If this is a Merge op, we automatically clear the list because

# a Merge node only requires one of its two inputs.

del recipient_pending_inputs[:]

else:

del recipient_pending_inputs[

recipient_pending_inputs.index((node, slot))]

def loaded_partition_graphs(self):

"""Test whether partition graphs have been loaded."""

return self._partition_graphs is not None

def partition_graphs(self):

"""Get the partition graphs.

if self._partition_graphs is None:

raise LookupError("No partition graphs have been loaded.")

return self._partition_graphs

def nodes(self):

"""Get a list of all nodes from the partition graphs.

if self._partition_graphs is None:

raise LookupError("No partition graphs have been loaded.")

return [node_name for node_name in self._node_inputs]

def node_attributes(self, node_name):

"""Get the attributes of a node.

if self._partition_graphs is None:

raise LookupError("No partition graphs have been loaded.")

if node_name in self._node_attributes:

return self._node_attributes[node_name]

else:

raise ValueError("No node named \"%s\" exists." % node_name)

def node_inputs(self, node_name, is_control=False):

"""Get the inputs of given node according to partition graphs.

if self._partition_graphs is None:

raise LookupError(

"Node inputs are not loaded from partition graphs yet.")

if node_name not in self._node_inputs:

raise ValueError("Node '%s' does not exist in partition graphs." %

node_name)

if is_control:

return self._node_ctrl_inputs[node_name]

else:

return self._node_inputs[node_name]

def transitive_inputs(self, node_name, include_control=True):

"""Get the transitive inputs of given node according to partition graphs.

if self._partition_graphs is None:

raise LookupError(

"Node inputs are not loaded from partition graphs yet.")

if node_name not in self._node_inputs:

raise ValueError("Node '%s' does not exist in partition graphs." %

node_name)

inputs = []

# Keep track of visited nodes to avoid infinite loops during input

# tracing.

visited_nodes = []

def trace_inputs(node):

"""Inner function for recursive tracing of node inputs.

node = get_node_name(node)

# Stop the tracing at a Merge op, as it is generally impossible to infer

# outside the runtime which input to the Merge op is alive.

if self._node_op_types[node] == "Merge":

return

if node in visited_nodes:

return

visited_nodes.append(node)

for inp in self._node_inputs[node]:

if inp == node_name:

continue

inputs.append(inp)

trace_inputs(inp)

if include_control:

for ctrl_inp in self._node_ctrl_inputs[node]:

if ctrl_inp == node_name:

continue

inputs.append(ctrl_inp)

trace_inputs(ctrl_inp)

trace_inputs(node_name)

return inputs

def node_recipients(self, node_name, is_control=False):

"""Get recipient of the given node's output according to partition graphs.

if self._partition_graphs is None:

raise LookupError(

"Node recipients are not loaded from partition graphs yet.")

if node_name not in self._node_recipients:

raise ValueError("Node '%s' does not exist in partition graphs." %

node_name)

if is_control:

return self._node_ctrl_recipients[node_name]

else:

return self._node_recipients[node_name]

def devices(self):

"""Get the list of devices.

if self._partition_graphs is None:

raise LookupError("Devices are not loaded from partition graphs yet.")

return self._devices

def node_exists(self, node_name):

"""Test if a node exists in the partition graphs.

if self._node_inputs is None: