: env_(env),

last_1000_steps_(1000),

step_count_(0),

session_gc_seconds_(session_gc_seconds) {

// Right now, a master service must be co-located with a device.

// Otherwise, fetches do not work.

CHECK(!env->local_devices.empty());

if (session_gc_seconds_ > 0.0) {

gc_thread_ = env_->env->StartThread(ThreadOptions(), "TF_master_GC",

[this]() { GC(); });

} else {

gc_thread_ = nullptr;

}

Env* env = Env::Default();

while (true) {

mutex_lock l(mu_);

const int kTimeoutMilliseconds = 10 * 1000; // 10 seconds.

WaitForMilliseconds(&l, &shutdown_cv_, kTimeoutMilliseconds);

if (shutdown_) {

break;

}

std::vector<string> handles;

const int64 num_micros = static_cast<int64>(session_gc_seconds_ * 1000000);

for (const auto& entry : sessions_) {

int64 lat = entry.second->last_access_time_usec();

if (static_cast<int64>(env->NowMicros()) - lat > num_micros) {

handles.push_back(entry.first);

auto* sess = entry.second;

SchedClosure([this, sess]() {

LOG(WARNING) << "GC session " << sess->handle() << " after "

<< session_gc_seconds_ << " seconds. "

<< "Note that if you are starting multiple replicas "

<< "on a staggered delay, session_gc_seconds may need "

<< "to be raised.";

sess->GarbageCollect();

});

}

}

for (const auto& handle : handles) sessions_.erase(handle);

}

public:

static Status GetRemoteDevices(

const protobuf::RepeatedPtrField<string>& device_filters, MasterEnv* env,

std::vector<Device*>* out_remote) {

DeviceFinder finder(device_filters, env);

finder.Start();

TF_RETURN_IF_ERROR(finder.Wait());

finder.GetRemoteDevices(env->local_devices, out_remote);

return Status::OK();

}

private:

explicit DeviceFinder(

const protobuf::RepeatedPtrField<string>& device_filters, MasterEnv* env)

: env_(env) {

auto process_filter = [this](const string& filter) {

DeviceNameUtils::ParsedName parsed;

if (DeviceNameUtils::ParseFullName(filter, &parsed)) {

filters_.push_back(parsed);

} else {

LOG(FATAL) << "Skipping invalid filter: " << filter;

}

};

for (const string& filter : device_filters) {

process_filter(filter);

}

// Enumerates all known workers' target. A target name is a

// prefix of a device name. E.g., /job:mnist/replica:0/task:10.

std::vector<string> workers;

env_->worker_cache->ListWorkers(&workers);

if (filters_.empty()) {

std::swap(workers, targets_);

} else {

for (const string& name : workers) {

if (MatchFilters(name)) {

targets_.push_back(name);

}

}

}

seen_targets_.assign(targets_.size(), false);

}

~DeviceFinder() {

for (Device* dev : found_) delete dev;

}

void Start() {

{

mutex_lock l(mu_);

num_pending_ = targets_.size();

if (num_pending_ == 0) {

pending_zero_.notify_all();

}

}

// Talk to all workers to get the list of available devices.

using std::placeholders::_1;

using std::placeholders::_2;

for (size_t i = 0; i < targets_.size(); ++i) {

// TODO(mrry): Propagate a timeout here, since `this->WhenFound()` may

// never be called.

NewRemoteDevices(env_->env, env_->worker_cache, targets_[i],

std::bind(&ME::WhenFound, this, i, _1, _2));

}

}

// Every `kLoggingPeriodMs`, while the DeviceFinder is still waiting

// to hear from workers, log a list of the workers who have not

// responded.

const int32 kLoggingPeriodMs = 10 * 1000;

Status Wait() {

mutex_lock l(mu_);

// TODO(mrry): Propagate a timeout here, since `num_pending_` may

// never become zero.

while (num_pending_ != 0) {

pending_zero_.wait_for(l, std::chrono::milliseconds(kLoggingPeriodMs));

if (num_pending_ != 0) {

for (int i = 0; i < targets_.size(); ++i) {

if (!seen_targets_[i]) {

LOG(INFO)

<< "CreateSession still waiting for response from worker: "

<< targets_[i];

}

}

}

}

return status_;

}

// The caller takes the ownership of returned remote devices.

void GetRemoteDevices(const std::vector<Device*>& local,

std::vector<Device*>* remote) {

std::unordered_set<string> names(local.size());

for (Device* dev : local) names.insert(dev->name());

mutex_lock l(mu_);

for (Device* dev : found_) {

const string& name = dev->name();

if (names.insert(name).second && MatchFilters(name)) {

remote->push_back(dev);

} else {

delete dev;

}

}

found_.clear();

}

typedef DeviceFinder ME;

const MasterEnv* env_;

std::vector<DeviceNameUtils::ParsedName> filters_;

mutex mu_;

int num_pending_ GUARDED_BY(mu_);

condition_variable pending_zero_;

std::vector<Device*> found_ GUARDED_BY(mu_);

// List of targets to be contacted by this DeviceFinder. The

// respective `bool` in `seen_targets_` indicates whether we have

// heard from this target or not.

std::vector<string> targets_;

std::vector<bool> seen_targets_ GUARDED_BY(mu_);

Status status_;

void WhenFound(int target_index, const Status& s,

std::vector<Device*>* devices) {

mutex_lock l(mu_);

seen_targets_[target_index] = true;

if (!s.ok()) {

LOG(ERROR) << "Master init: " << s;

status_.Update(s);

} else {

found_.insert(found_.end(), devices->begin(), devices->end());

devices->clear();

}

--num_pending_;

if (num_pending_ == 0) {

pending_zero_.notify_all();

}

}

// Returns true iff the set of devices allowed by 'x' intersects

// with the set of devices allowed by 'y'.

bool Intersects(const DeviceNameUtils::ParsedName& x,

const DeviceNameUtils::ParsedName& y) {

return (!x.has_job || !y.has_job || x.job == y.job) &&

(!x.has_replica || !y.has_replica || x.replica == y.replica) &&

(!x.has_task || !y.has_task || x.task == y.task) &&

(!x.has_type || !y.has_type || x.type == y.type) &&

(!x.has_id || !y.has_id || x.id == y.id);

}

// Returns true iff 'name' matches one of the filters_.

bool MatchFilters(const string& name) {

if (filters_.empty()) return true;

DeviceNameUtils::ParsedName x;

if (DeviceNameUtils::ParseFullName(name, &x)) {

for (const auto& filter : filters_) {

if (Intersects(x, filter)) return true;

}

}

return false;

}

TF_DISALLOW_COPY_AND_ASSIGN(DeviceFinder);

CreateSessionResponse* resp, MyClosure done) {

SchedClosure([this, req, resp, done]() {

Status status = ValidateExternalGraphDefSyntax(req->graph_def());

if (status.ok()) {

// Ping all the workers and build the list of devices that the

// session will use.

std::vector<Device*> remote_devices;

status = DeviceFinder::GetRemoteDevices(req->config().device_filters(),

env_, &remote_devices);

if (!status.ok()) {

done(status);

return;

}

SessionOptions options;

options.config = req->config();

MasterSession* session =

env_->master_session_factory(options, env_, &remote_devices);

GraphDef* gdef =

const_cast<CreateSessionRequest*>(req)->mutable_graph_def();

Status create_status = session->Create(gdef);

if (!create_status.ok()) {

session->Close().IgnoreError();

session->Unref();

done(create_status);

return;

}

resp->set_session_handle(session->handle());

// Insert into the session map, which takes ownership of the session.

{

mutex_lock l(mu_);

CHECK(sessions_.insert({session->handle(), session}).second);

}

}

done(status);

});

ExtendSessionResponse* resp, MyClosure done) {

mu_.lock();

MasterSession* session = nullptr;

session = gtl::FindPtrOrNull(sessions_, req->session_handle());

if (session == nullptr) {

mu_.unlock();

done(errors::Aborted("Session ", req->session_handle(), " is not found."));

return;

}

session->Ref();

mu_.unlock();

SchedClosure([session, req, resp, done]() {

Status status = ValidateExternalGraphDefSyntax(req->graph_def());

if (status.ok()) {

status = session->Extend(req, resp);

}

session->Unref();

done(status);

});

PartialRunSetupResponse* resp, MyClosure done) {

mu_.lock();

MasterSession* session = gtl::FindPtrOrNull(sessions_, req->session_handle());

if (session == nullptr) {

mu_.unlock();

done(errors::Aborted("Session ", req->session_handle(), " is not found."));

return;

}

session->Ref();

mu_.unlock();

SchedClosure([this, session, req, resp, done]() {

Status s = session->PartialRunSetup(req, resp);

session->Unref();

done(s);

});

MutableRunStepResponseWrapper* resp, MyClosure done) {

mu_.lock();

uint64 start_time = env_->env->NowMicros();

MasterSession* session = gtl::FindPtrOrNull(sessions_, req->session_handle());

if (session == nullptr) {

mu_.unlock();

done(errors::Aborted("Session ", req->session_handle(), " is not found."));

return;

}

session->Ref();

mu_.unlock();

SchedClosure([this, start_time, session, opts, req, resp, done]() {

Status status = session->Run(opts, *req, resp);

session->Unref();

uint64 done_time = env_->env->NowMicros();

done(status);

mutex_lock l(mu_);

last_1000_steps_.AddValue((done_time - start_time) / 1e9);

++step_count_;

});

CloseSessionResponse* resp, MyClosure done) {

MasterSession* session = nullptr;

{

mu_.lock();

auto iter = sessions_.find(req->session_handle());

if (iter == sessions_.end()) {

mu_.unlock();

done(errors::Aborted(

"Session ", req->session_handle(),

" is not found. Possibly, this master has restarted."));

return;

}

// NOTE(mrry): One reference to the session is transferred from

// `sessions_[req->session_handle()]` to `session`.

session = iter->second;

sessions_.erase(iter);

mu_.unlock();

}

// Session Close() blocks on thread shutdown. Therefore, we need to

// delete it in non-critical thread.

SchedClosure([session, done]() {

Status s = session->Close();

session->Unref();

done(s);

});

ListDevicesResponse* resp, MyClosure done) {

SchedClosure([this, req, resp, done]() {

std::vector<Device*> remote_devices;

Status s = DeviceFinder::GetRemoteDevices({}, env_, &remote_devices);

if (s.ok()) {

for (Device* dev : env_->local_devices) {

*(resp->add_local_device()) = dev->attributes();

}

for (Device* dev : remote_devices) {

*(resp->add_remote_device()) = dev->attributes();

delete dev;

}

}

done(s);

});

std::vector<string> worker_names;

env_->worker_cache->ListWorkers(&worker_names);

if (!worker_names.empty()) {

const int num_workers = worker_names.size();

std::vector<Notification> n(num_workers);

CleanupAllRequest req;

(*req.mutable_container()) = reset.container();

std::vector<CleanupAllResponse> resp(num_workers);

int c = 0;

for (int i = 0; i < num_workers; ++i) {

const string& worker_name = worker_names[i];

auto worker = env_->worker_cache->CreateWorker(worker_name);

if (worker) {

worker->CleanupAllAsync(

&req, &resp[i], [this, &n, worker_name, worker, c](Status s) {

TF_CHECK_OK(s);

env_->worker_cache->ReleaseWorker(worker_name, worker);

n[c].Notify();

});

} else {

n[c].Notify();

}

++c;

}

for (size_t i = 0; i < n.size(); ++i) {

n[i].WaitForNotification();

}

}

MyClosure done) {

// Vector to hold the session pointers present in the sessions_

// (string->Session*) map.

std::vector<MasterSession*> sessions_to_close;

{

mutex_lock l(mu_);

// NOTE(mrry): Transfer one reference to each session from the

// `sessions_` map to the `sessions_to_close` vector.

for (const auto& entry : sessions_) {

sessions_to_close.push_back(entry.second);

}

sessions_.clear();

}

CleanupWorkers(*req);

SchedClosure([sessions_to_close, done]() {

Status s;

for (MasterSession* session : sessions_to_close) {

s.Update(session->Close());

session->Unref();

}

done(s);

});