forked from grafana-cold-storage/metrictank
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathplan.go
More file actions
240 lines (220 loc) · 7.59 KB
/
Copy pathplan.go
File metadata and controls
240 lines (220 loc) · 7.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
package expr
import (
"errors"
"fmt"
"io"
"sort"
"github.com/raintank/metrictank/api/models"
"github.com/raintank/metrictank/consolidation"
)
// Req represents a request for one/more series
type Req struct {
Query string // whatever was parsed as the query out of a graphite target. e.g. target=sum(foo.{b,a}r.*) -> foo.{b,a}r.* -> this will go straight to index lookup
From uint32
To uint32
Cons consolidation.Consolidator // can be 0 to mean undefined
}
// NewReq creates a new Req. pass cons=0 to leave consolidator undefined,
// leaving up to the caller (in graphite's case, it would cause a lookup into storage-aggregation.conf)
func NewReq(query string, from, to uint32, cons consolidation.Consolidator) Req {
return Req{
Query: query,
From: from,
To: to,
Cons: cons,
}
}
type Plan struct {
Reqs []Req // data that needs to be fetched before functions can be executed
funcs []GraphiteFunc // top-level funcs to execute, the head of each tree for each target
exprs []*expr
MaxDataPoints uint32
From uint32 // global request scoped from
To uint32 // global request scoped to
data map[Req][]models.Series // input data to work with. set via Run(), as well as
// new data generated by processing funcs. useful for two reasons:
// 1) reuse partial calculations e.g. queries like target=movingAvg(sum(foo), 10)&target=sum(foo) (TODO)
// 2) central place to return data back to pool when we're done.
}
func (p Plan) Dump(w io.Writer) {
fmt.Fprintf(w, "Plan:\n")
fmt.Fprintf(w, "* Exprs:\n")
for _, e := range p.exprs {
fmt.Fprintln(w, e.Print(2))
}
fmt.Fprintf(w, "* Reqs:\n")
for _, r := range p.Reqs {
fmt.Fprintln(w, " ", r)
}
fmt.Fprintf(w, "MaxDataPoints: %d\n", p.MaxDataPoints)
fmt.Fprintf(w, "From: %d\n", p.From)
fmt.Fprintf(w, "To: %d\n", p.To)
}
// Plan validates the expressions and comes up with the initial (potentially non-optimal) execution plan
// which is just a list of requests and the expressions.
// traverse tree and as we go down:
// * make sure function exists
// * validation of arguments
// * allow functions to modify the Context (change data range or consolidation)
// * future version: allow functions to mark safe to pre-aggregate using consolidateBy or not
func NewPlan(exprs []*expr, from, to, mdp uint32, stable bool, reqs []Req) (Plan, error) {
var err error
var funcs []GraphiteFunc
for _, e := range exprs {
var fn GraphiteFunc
context := Context{
from: from,
to: to,
}
fn, reqs, err = newplan(e, context, stable, reqs)
if err != nil {
return Plan{}, err
}
funcs = append(funcs, fn)
}
return Plan{
Reqs: reqs,
exprs: exprs,
funcs: funcs,
MaxDataPoints: mdp,
From: from,
To: to,
}, nil
}
// newplan adds requests as needed for the given expr, resolving function calls as needed
func newplan(e *expr, context Context, stable bool, reqs []Req) (GraphiteFunc, []Req, error) {
if e.etype != etFunc && e.etype != etName {
return nil, nil, errors.New("request must be a function call or metric pattern")
}
if e.etype == etName {
req := NewReq(e.str, context.from, context.to, context.consol)
reqs = append(reqs, req)
return NewGet(req), reqs, nil
}
// here e.type is guaranteed to be etFunc
fdef, ok := funcs[e.str]
if !ok {
return nil, nil, ErrUnknownFunction(e.str)
}
if stable && !fdef.stable {
return nil, nil, ErrUnknownFunction(e.str)
}
fn := fdef.constr()
reqs, err := newplanFunc(e, fn, context, stable, reqs)
return fn, reqs, err
}
// newplanFunc adds requests as needed for the given expr, and validates the function input
// provided you already know the expression is a function call to the given function
func newplanFunc(e *expr, fn GraphiteFunc, context Context, stable bool, reqs []Req) ([]Req, error) {
// first comes the interesting task of validating the arguments as specified by the function,
// against the arguments that were parsed.
argsExp, _ := fn.Signature()
var err error
// note:
// * signature may have seriesLists in it, which means one or more args of type seriesList
// so it's legal to have more e.args than signature args in that case.
// * we can't do extensive, accurate validation of the type here because what the output from a function we depend on
// might be dynamically typed. e.g. movingAvg returns 1..N series depending on how many it got as input
// first validate the mandatory args
pos := 0 // pos in args of next given arg to process
cutoff := 0 // marks the index of the first optional point (if any)
var argExp Arg
for cutoff, argExp = range argsExp {
if argExp.Optional() {
break
}
if len(e.args) <= pos {
return nil, ErrMissingArg
}
pos, err = e.consumeBasicArg(pos, argExp)
if err != nil {
return nil, err
}
}
if !argExp.Optional() {
cutoff += 1
}
// we stopped iterating the mandatory args.
// any remaining args should be due to optional args otherwise there's too many
// we also track here which keywords can also be used for the given optional args
// so that those args should not be specified via their keys anymore.
seenKwargs := make(map[string]struct{})
for _, argOpt := range argsExp[cutoff:] {
if len(e.args) <= pos {
break // no more args specified. we're done.
}
pos, err = e.consumeBasicArg(pos, argOpt)
if err != nil {
return nil, err
}
seenKwargs[argOpt.Key()] = struct{}{}
}
if len(e.args) > pos {
return nil, ErrTooManyArg
}
// for any provided keyword args, verify that they are what the function stipulated
// and that they have not already been specified via their position
for key := range e.namedArgs {
_, ok := seenKwargs[key]
if ok {
return nil, ErrKwargSpecifiedTwice{key}
}
err = e.consumeKwarg(key, argsExp[cutoff:])
if err != nil {
return nil, err
}
seenKwargs[key] = struct{}{}
}
// functions now have their non-series input args set,
// so they should now be able to specify any context alterations
context = fn.Context(context)
// now that we know the needed context for the data coming into
// this function, we can set up the input arguments for the function
// that are series
pos = 0
for _, argExp = range argsExp[:cutoff] {
switch argExp.(type) {
case ArgSeries, ArgSeriesList, ArgSeriesLists:
pos, reqs, err = e.consumeSeriesArg(pos, argExp, context, stable, reqs)
if err != nil {
return nil, err
}
default:
return reqs, err
}
}
return reqs, err
}
// Run invokes all processing as specified in the plan (expressions, from/to) with the input as input
func (p Plan) Run(input map[Req][]models.Series) ([]models.Series, error) {
var out []models.Series
p.data = input
for _, fn := range p.funcs {
series, err := fn.Exec(p.data)
if err != nil {
return nil, err
}
sort.Sort(models.SeriesByTarget(series))
out = append(out, series...)
}
for i, o := range out {
if p.MaxDataPoints != 0 && len(o.Datapoints) > int(p.MaxDataPoints) {
// series may have been created by a function that didn't know which consolidation function to default to.
// in the future maybe we can do more clever things here. e.g. perSecond maybe consolidate by max.
if o.Consolidator == 0 {
o.Consolidator = consolidation.Avg
}
out[i].Datapoints, out[i].Interval = consolidation.ConsolidateStable(o.Datapoints, o.Interval, p.MaxDataPoints, o.Consolidator)
}
}
return out, nil
}
// Clean returns all buffers (all input data + generated series along the way)
// back to the pool.
func (p Plan) Clean() {
for _, series := range p.data {
for _, serie := range series {
pointSlicePool.Put(serie.Datapoints[:0])
}
}
}