forked from tensorflow/tensorflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdebug_io_utils.cc
More file actions
211 lines (177 loc) · 7.63 KB
/
Copy pathdebug_io_utils.cc
File metadata and controls
211 lines (177 loc) · 7.63 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/debug/debug_io_utils.h"
#include <vector>
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/util/event.pb.h"
namespace tensorflow {
namespace {
// Encapsulate the tensor value inside a Summary proto, and then inside an
// Event proto.
Event WrapTensorAsEvent(const string& tensor_name, const string& debug_op,
const Tensor& tensor, const uint64 wall_time_us) {
Event event;
event.set_wall_time(static_cast<double>(wall_time_us));
Summary::Value* summ_val = event.mutable_summary()->add_value();
// Create the debug node_name in the Summary proto.
// For example, if tensor_name = "foo/node_a:0", and the debug_op is
// "DebugIdentity", the debug node_name in the Summary proto will be
// "foo/node_a:0:DebugIdentity".
const string debug_node_name = strings::StrCat(tensor_name, ":", debug_op);
summ_val->set_node_name(debug_node_name);
if (tensor.dtype() == DT_STRING) {
// Treat DT_STRING specially, so that tensor_util.MakeNdarray can convert
// the TensorProto to string-type numpy array. MakeNdarray does not work
// with strings encoded by AsProtoTensorContent() in tensor_content.
tensor.AsProtoField(summ_val->mutable_tensor());
} else {
tensor.AsProtoTensorContent(summ_val->mutable_tensor());
}
return event;
}
} // namespace
// static
const char* const DebugIO::kFileURLScheme = "file://";
// static
const char* const DebugIO::kGrpcURLScheme = "grpc://";
Status DebugIO::PublishDebugTensor(const string& tensor_name,
const string& debug_op, const Tensor& tensor,
const uint64 wall_time_us,
const gtl::ArraySlice<string>& debug_urls) {
// Split the tensor_name into node name and output slot index.
std::vector<string> name_items = str_util::Split(tensor_name, ':');
string node_name;
int32 output_slot = 0;
if (name_items.size() == 2) {
node_name = name_items[0];
if (!strings::safe_strto32(name_items[1], &output_slot)) {
return Status(error::INVALID_ARGUMENT,
strings::StrCat("Invalid string value for output_slot: \"",
name_items[1], "\""));
}
} else if (name_items.size() == 1) {
node_name = name_items[0];
} else {
return Status(
error::INVALID_ARGUMENT,
strings::StrCat("Failed to parse tensor name: \"", tensor_name, "\""));
}
int num_failed_urls = 0;
for (const string& url : debug_urls) {
if (str_util::Lowercase(url).find(kFileURLScheme) == 0) {
const string dump_root_dir = url.substr(strlen(kFileURLScheme));
Status s =
DebugFileIO::DumpTensorToDir(node_name, output_slot, debug_op, tensor,
wall_time_us, dump_root_dir, nullptr);
if (!s.ok()) {
num_failed_urls++;
}
} else if (str_util::Lowercase(url).find(kGrpcURLScheme) == 0) {
// TODO(cais): Implement PublishTensor with grpc urls.
return Status(error::UNIMPLEMENTED,
strings::StrCat("Puslishing to GRPC debug target is not ",
"implemented yet"));
} else {
return Status(error::UNAVAILABLE,
strings::StrCat("Invalid debug target URL: ", url));
}
}
if (num_failed_urls == 0) {
return Status::OK();
} else {
return Status(
error::INTERNAL,
strings::StrCat("Puslishing to ", num_failed_urls, " of ",
debug_urls.size(), " debug target URLs failed"));
}
}
// static
Status DebugFileIO::DumpTensorToDir(
const string& node_name, const int32 output_slot, const string& debug_op,
const Tensor& tensor, const uint64 wall_time_us,
const string& dump_root_dir, string* dump_file_path) {
const string file_path = GetDumpFilePath(dump_root_dir, node_name,
output_slot, debug_op, wall_time_us);
if (dump_file_path != nullptr) {
*dump_file_path = file_path;
}
return DumpTensorToEventFile(node_name, output_slot, debug_op, tensor,
wall_time_us, file_path);
}
// static
string DebugFileIO::GetDumpFilePath(const string& dump_root_dir,
const string& node_name,
const int32 output_slot,
const string& debug_op,
const uint64 wall_time_us) {
return io::JoinPath(
dump_root_dir, strings::StrCat(node_name, "_", output_slot, "_", debug_op,
"_", wall_time_us));
}
// static
Status DebugFileIO::DumpTensorToEventFile(
const string& node_name, const int32 output_slot, const string& debug_op,
const Tensor& tensor, const uint64 wall_time_us, const string& file_path) {
Env* env(Env::Default());
// Create the directory if necessary.
string file_dir = io::Dirname(file_path).ToString();
Status s = DebugFileIO::RecursiveCreateDir(env, file_dir);
if (!s.ok()) {
return Status(error::FAILED_PRECONDITION,
strings::StrCat("Failed to create directory ", file_dir,
", due to: ", s.error_message()));
}
const string tensor_name = strings::StrCat(node_name, ":", output_slot);
Event event = WrapTensorAsEvent(tensor_name, debug_op, tensor, wall_time_us);
string event_str;
event.SerializeToString(&event_str);
std::unique_ptr<WritableFile> f = nullptr;
TF_CHECK_OK(env->NewWritableFile(file_path, &f));
f->Append(event_str);
TF_CHECK_OK(f->Close());
return Status::OK();
}
// static
Status DebugFileIO::RecursiveCreateDir(Env* env, const string& dir) {
if (env->FileExists(dir) && env->IsDirectory(dir).ok()) {
// The path already exists as a directory. Return OK right away.
return Status::OK();
}
string parent_dir = io::Dirname(dir).ToString();
if (!env->FileExists(parent_dir)) {
// The parent path does not exist yet, create it first.
Status s = RecursiveCreateDir(env, parent_dir); // Recursive call
if (!s.ok()) {
return Status(
error::FAILED_PRECONDITION,
strings::StrCat("Failed to create directory ", parent_dir));
}
} else if (env->FileExists(parent_dir) &&
!env->IsDirectory(parent_dir).ok()) {
// The path exists, but it is a file.
return Status(error::FAILED_PRECONDITION,
strings::StrCat("Failed to create directory ", parent_dir,
" because the path exists as a file "));
}
env->CreateDir(dir);
// Guard against potential race in creating directories by doing a check
// after the CreateDir call.
if (env->FileExists(dir) && env->IsDirectory(dir).ok()) {
return Status::OK();
} else {
return Status(error::ABORTED,
strings::StrCat("Failed to create directory ", parent_dir));
}
}
} // namespace tensorflow