-
Notifications
You must be signed in to change notification settings - Fork 148
Expand file tree
/
Copy pathlambdaManager.go
More file actions
206 lines (170 loc) · 5.56 KB
/
lambdaManager.go
File metadata and controls
206 lines (170 loc) · 5.56 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
package lambda
import (
"container/list"
"log"
"net/http"
"path/filepath"
"strings"
"sync"
"github.com/open-lambda/open-lambda/ol/common"
"github.com/open-lambda/open-lambda/ol/worker/lambda/packages"
"github.com/open-lambda/open-lambda/ol/worker/lambda/zygote"
"github.com/open-lambda/open-lambda/ol/worker/sandbox"
)
// LambdaMgr provides thread-safe getting of lambda functions and collects all
// lambda subsystems (resource pullers and sandbox pools) in one place
type LambdaMgr struct {
// subsystems (these are thread safe)
sbPool sandbox.SandboxPool
*packages.DepTracer
*packages.PackagePuller // depends on sbPool and DepTracer
zygote.ZygoteProvider // depends PackagePuller
*HandlerPuller // depends on sbPool and ImportCache[optional]
// storage dirs that we manage
codeDirs *common.DirMaker
scratchDirs *common.DirMaker
// thread-safe map from a lambda's name to its LambdaFunc
mapMutex sync.Mutex
lfuncMap map[string]*LambdaFunc
}
// represents an HTTP request to be handled by a lambda instance
type Invocation struct {
w http.ResponseWriter
r *http.Request
// signal to client that response has been written to w
done chan bool
// how many milliseconds did ServeHTTP take? (doesn't count
// queue time or Sandbox init)
execMs int
}
func NewLambdaMgr() (res *LambdaMgr, err error) {
mgr := &LambdaMgr{
lfuncMap: make(map[string]*LambdaFunc),
}
defer func() {
if err != nil {
log.Printf("Cleanup Lambda Manager due to error: %v", err)
mgr.Cleanup()
}
}()
mgr.codeDirs, err = common.NewDirMaker("code", common.Conf.Storage.Code.Mode())
if err != nil {
return nil, err
}
mgr.scratchDirs, err = common.NewDirMaker("scratch", common.Conf.Storage.Scratch.Mode())
if err != nil {
return nil, err
}
log.Printf("Creating SandboxPool")
mgr.sbPool, err = sandbox.SandboxPoolFromConfig("sandboxes", common.Conf.Mem_pool_mb)
if err != nil {
return nil, err
}
log.Printf("Creating DepTracer")
mgr.DepTracer, err = packages.NewDepTracer(filepath.Join(common.Conf.Worker_dir, "dep-trace.json"))
if err != nil {
return nil, err
}
log.Printf("Creating PackagePuller")
mgr.PackagePuller, err = packages.NewPackagePuller(mgr.sbPool, mgr.DepTracer)
if err != nil {
return nil, err
}
if common.Conf.Features.Import_cache != "" {
log.Printf("Creating ImportCache")
mgr.ZygoteProvider, err = zygote.NewZygoteProvider(mgr.codeDirs, mgr.scratchDirs, mgr.sbPool, mgr.PackagePuller)
if err != nil {
return nil, err
}
}
log.Printf("Creating HandlerPuller")
mgr.HandlerPuller, err = NewHandlerPuller(mgr.codeDirs)
if err != nil {
return nil, err
}
return mgr, nil
}
// Returns an existing instance (if there is one), or creates a new one
func (mgr *LambdaMgr) Get(name string) (f *LambdaFunc) {
mgr.mapMutex.Lock()
defer mgr.mapMutex.Unlock()
f = mgr.lfuncMap[name]
if f == nil {
f = &LambdaFunc{
lmgr: mgr,
name: name,
// TODO make these configurable
funcChan: make(chan *Invocation, 1024),
instChan: make(chan *Invocation, 1024),
doneChan: make(chan *Invocation, 1024),
instances: list.New(),
killChan: make(chan chan bool, 1),
}
go f.Task()
mgr.lfuncMap[name] = f
}
return f
}
func (mgr *LambdaMgr) Debug() string {
return mgr.sbPool.DebugString() + "\n"
}
func (_ *LambdaMgr) DumpStatsToLog() {
snapshot := common.SnapshotStats()
sec := func(name string) float64 {
return float64(snapshot[name+".cnt"]*snapshot[name+".ms-avg"]) / 1000
}
time := func(indent int, name string, parent string) {
selftime := sec(name)
ptime := sec(parent)
tabs := strings.Repeat("\t", indent)
if ptime > 0 {
log.Printf("%s%s: %.3f (%.1f%%)", tabs, name, selftime, selftime/ptime*100)
} else {
log.Printf("%s%s: %.3f", tabs, name, selftime)
}
}
log.Printf("Request Profiling (cumulative seconds):")
time(0, "LambdaFunc.Invoke", "")
time(1, "LambdaInstance-WaitSandbox", "LambdaFunc.Invoke")
time(2, "LambdaInstance-WaitSandbox-Unpause", "LambdaInstance-WaitSandbox")
time(2, "LambdaInstance-WaitSandbox-NoImportCache", "LambdaInstance-WaitSandbox")
time(2, "ImportCache.Create", "LambdaInstance-WaitSandbox")
time(3, "ImportCache.root.Lookup", "ImportCache.Create")
time(3, "ImportCache.createChildSandboxFromNode", "ImportCache.Create")
time(4, "ImportCache.getSandboxInNode", "ImportCache.createChildSandboxFromNode")
time(4, "ImportCache.createChildSandboxFromNode:childSandboxPool.Create",
"ImportCache.createChildSandboxFromNode")
time(4, "ImportCache.putSandboxInNode", "ImportCache.createChildSandboxFromNode")
time(5, "ImportCache.putSandboxInNode:Lock", "ImportCache.putSandboxInNode")
time(5, "ImportCache.putSandboxInNode:Pause", "ImportCache.putSandboxInNode")
time(1, "LambdaInstance-ServeRequests", "LambdaFunc.Invoke")
time(2, "LambdaInstance-RoundTrip", "LambdaInstance-ServeRequests")
}
func (mgr *LambdaMgr) Cleanup() {
mgr.mapMutex.Lock() // don't unlock, because this shouldn't be used anymore
mgr.DumpStatsToLog()
// HandlerPuller+PackagePuller requires no cleanup
// 1. cleanup handler Sandboxes
// 2. cleanup Zygote Sandboxes (after the handlers, which depend on the Zygotes)
// 3. cleanup SandboxPool underlying both of above
for _, f := range mgr.lfuncMap {
log.Printf("Kill function: %s", f.name)
f.Kill()
}
if mgr.ZygoteProvider != nil {
mgr.ZygoteProvider.Cleanup()
}
if mgr.sbPool != nil {
mgr.sbPool.Cleanup() // assumes all Sandboxes are gone
}
// cleanup DepTracer
if mgr.DepTracer != nil {
mgr.DepTracer.Cleanup()
}
if mgr.codeDirs != nil {
mgr.codeDirs.Cleanup()
}
if mgr.scratchDirs != nil {
mgr.scratchDirs.Cleanup()
}
}