Repository navigation
Expand file tree
/
Copy pathinfer_controller.hpp
More file actions
154 lines (127 loc) · 4.37 KB
/
Copy pathinfer_controller.hpp
File metadata and controls
154 lines (127 loc) · 4.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
#ifndef INFER_CONTROLLER_HPP
#define INFER_CONTROLLER_HPP
#include <string>
#include <future>
#include <memory>
#include <mutex>
#include <thread>
#include <queue>
#include <condition_variable>
#include <infer/trt_infer.hpp>
#include "monopoly_allocator.hpp"
template<class Input, class Output, class StartParam=std::tuple<std::string, int>, class JobAdditional=int>
class InferController{
public:
struct Job{
Input input;
Output output;
JobAdditional additional;
MonopolyAllocator<TRT::Tensor>::MonopolyDataPointer mono_tensor;
std::shared_ptr<std::promise<Output>> pro;
};
virtual ~InferController(){
stop();
}
void stop(){
run_ = false;
cond_.notify_all();
////////////////////////////////////////// cleanup jobs
{
std::unique_lock<std::mutex> l(jobs_lock_);
while(!jobs_.empty()){
auto& item = jobs_.front();
if(item.pro)
item.pro->set_value(Output());
jobs_.pop();
}
};
if(worker_){
worker_->join();
worker_.reset();
}
}
bool startup(const StartParam& param){
run_ = true;
std::promise<bool> pro;
start_param_ = param;
worker_ = std::make_shared<std::thread>(&InferController::worker, this, std::ref(pro));
return pro.get_future().get();
}
virtual std::shared_future<Output> commit(const Input& input){
Job job;
job.pro = std::make_shared<std::promise<Output>>();
if(!preprocess(job, input)){
job.pro->set_value(Output());
return job.pro->get_future();
}
///////////////////////////////////////////////////////////
{
std::unique_lock<std::mutex> l(jobs_lock_);
jobs_.push(job);
};
cond_.notify_one();
return job.pro->get_future();
}
virtual std::vector<std::shared_future<Output>> commits(const std::vector<Input>& inputs){
int batch_size = std::min((int)inputs.size(), this->tensor_allocator_->capacity());
std::vector<Job> jobs(inputs.size());
std::vector<std::shared_future<Output>> results(inputs.size());
int nepoch = (inputs.size() + batch_size - 1) / batch_size;
for(int epoch = 0; epoch < nepoch; ++epoch){
int begin = epoch * batch_size;
int end = std::min((int)inputs.size(), begin + batch_size);
for(int i = begin; i < end; ++i){
Job& job = jobs[i];
job.pro = std::make_shared<std::promise<Output>>();
if(!preprocess(job, inputs[i])){
job.pro->set_value(Output());
}
results[i] = job.pro->get_future();
}
///////////////////////////////////////////////////////////
{
std::unique_lock<std::mutex> l(jobs_lock_);
for(int i = begin; i < end; ++i){
jobs_.emplace(std::move(jobs[i]));
};
}
cond_.notify_one();
}
return results;
}
protected:
virtual void worker(std::promise<bool>& result) = 0;
virtual bool preprocess(Job& job, const Input& input) = 0;
virtual bool get_jobs_and_wait(std::vector<Job>& fetch_jobs, int max_size){
std::unique_lock<std::mutex> l(jobs_lock_);
cond_.wait(l, [&](){
return !run_ || !jobs_.empty();
});
if(!run_) return false;
fetch_jobs.clear();
for(int i = 0; i < max_size && !jobs_.empty(); ++i){
fetch_jobs.emplace_back(std::move(jobs_.front()));
jobs_.pop();
}
return true;
}
virtual bool get_job_and_wait(Job& fetch_job){
std::unique_lock<std::mutex> l(jobs_lock_);
cond_.wait(l, [&](){
return !run_ || !jobs_.empty();
});
if(!run_) return false;
fetch_job = std::move(jobs_.front());
jobs_.pop();
return true;
}
protected:
StartParam start_param_;
std::atomic<bool> run_;
std::mutex jobs_lock_;
std::queue<Job> jobs_;
std::shared_ptr<std::thread> worker_;
std::condition_variable cond_;
std::shared_ptr<MonopolyAllocator<TRT::Tensor>> tensor_allocator_;
};
#endif // INFER_CONTROLLER_HPP