Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions central/config/service/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/stackrox/rox/central/config/datastore"
"github.com/stackrox/rox/central/convert/storagetov1"
"github.com/stackrox/rox/central/convert/v1tostorage"
customMetrics "github.com/stackrox/rox/central/metrics/aggregator"
"github.com/stackrox/rox/central/platform/matcher"
"github.com/stackrox/rox/central/platform/reprocessor"
"github.com/stackrox/rox/central/telemetry/centralclient"
Expand Down Expand Up @@ -131,6 +132,9 @@ func (s *serviceImpl) GetConfig(ctx context.Context, _ *v1.Empty) (*storage.Conf

// PutConfig updates Central's config
func (s *serviceImpl) PutConfig(ctx context.Context, req *v1.PutConfigRequest) (*storage.Config, error) {

// Validation:

if req.GetConfig() == nil {
return nil, errors.Wrap(errox.InvalidArgs, "config must be specified")
}
Expand Down Expand Up @@ -160,16 +164,28 @@ func (s *serviceImpl) PutConfig(ctx context.Context, req *v1.PutConfigRequest) (
regexes = append(regexes, regex)
}
}

if err := customMetrics.ValidateConfiguration(
req.GetConfig().GetPrivateConfig().GetMetrics()); err != nil {
return nil, err
}

// Store:

if err := s.datastore.UpsertConfig(ctx, req.GetConfig()); err != nil {
return nil, err
}

// Application:

if req.GetConfig().GetPublicConfig().GetTelemetry().GetEnabled() {
centralclient.Enable()
} else {
centralclient.Disable()
}
matcher.Singleton().SetRegexes(regexes)
go reprocessor.Singleton().RunReprocessor()

return req.GetConfig(), nil
}

Expand Down
78 changes: 78 additions & 0 deletions central/metrics/aggregator/common/aggregator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package common

import (
"iter"

"github.com/prometheus/client_golang/prometheus"
)

type aggregationKey string // e.g. IMPORTANT_VULNERABILITY_SEVERITY|true

// aggregatedRecord is a single gauge metric record.
type aggregatedRecord struct {
labels prometheus.Labels
total int
}

// aggregator computes the aggregation result.
type aggregator[Finding any] struct {
result map[MetricName]map[aggregationKey]*aggregatedRecord
mcfg MetricsConfiguration
labelOrder map[Label]int
getters map[Label]func(Finding) string
}

func makeAggregator[Finding any](mcfg MetricsConfiguration, labelOrder map[Label]int, getters map[Label]func(Finding) string) *aggregator[Finding] {
aggregated := make(map[MetricName]map[aggregationKey]*aggregatedRecord)
for metric := range mcfg {
aggregated[metric] = make(map[aggregationKey]*aggregatedRecord)
}
return &aggregator[Finding]{aggregated, mcfg, labelOrder, getters}
}

// count the finding in the aggregation result.
func (r *aggregator[Finding]) count(finding Finding) {
labelValue := func(label Label) string {
return r.getters[label](finding)
}
for metric, labels := range r.mcfg {
if key, labels := makeAggregationKey(labels, labelValue, r.labelOrder); key != "" {
if rec, ok := r.result[metric][key]; ok {
rec.total++
} else {
r.result[metric][key] = &aggregatedRecord{labels, 1}
}
}
}
}

// makeAggregationKey computes an aggregation key according to the labels from
// the provided expression, and the map of the requested labels to their values.
// The values in the key are sorted according to the provided labelOrder map.
//
// Example:
//
// "Cluster,Deployment" => "pre-prod|backend", {"Cluster": "pre-prod", "Deployment": "backend")}
func makeAggregationKey(labelExpression []Label, getter func(Label) string, labelOrder map[Label]int) (aggregationKey, prometheus.Labels) {
labels := make(prometheus.Labels)
values := make(orderedValues, len(labelExpression))
for label, value := range collectMatchingLabels(labelExpression, getter) {
labels[string(label)] = value
values = append(values, valueOrder{labelOrder[label], value})
}
if len(labels) != len(labelExpression) {
return "", nil
}
return aggregationKey(values.join('|')), labels
}

// collectMatchingLabels returns an iterator over the labels and the values.
func collectMatchingLabels(labels []Label, getter func(Label) string) iter.Seq2[Label, string] {
return func(yield func(Label, string) bool) {
for _, label := range labels {
if !yield(label, getter(label)) {
return
}
}
}
}
141 changes: 141 additions & 0 deletions central/metrics/aggregator/common/aggregator_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package common

import (
"testing"

"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
)

func Test_aggregator(t *testing.T) {
getters := map[Label]func(testFinding) string{
"Severity": func(tf testFinding) string { return testData[tf]["Severity"] },
"Cluster": func(tf testFinding) string { return testData[tf]["Cluster"] },
"Namespace": func(tf testFinding) string { return testData[tf]["Namespace"] },
}
a := makeAggregator(makeTestMetricLabelExpression(t), testLabelOrder, getters)
assert.NotNil(t, a)
assert.Equal(t, map[MetricName]map[aggregationKey]*aggregatedRecord{
"Test_aggregator_metric1": {},
"Test_aggregator_metric2": {},
}, a.result)

a.count(testFinding(0))

assert.Equal(t, map[MetricName]map[aggregationKey]*aggregatedRecord{
"Test_aggregator_metric1": {
"cluster 1|CRITICAL": &aggregatedRecord{
labels: prometheus.Labels{"Cluster": testData[0]["Cluster"], "Severity": testData[0]["Severity"]},
total: 1,
}},
"Test_aggregator_metric2": {
"ns 1": &aggregatedRecord{
labels: prometheus.Labels{"Namespace": testData[0]["Namespace"]},
total: 1,
}},
}, a.result)

a.count(testFinding(0))

assert.Equal(t, map[MetricName]map[aggregationKey]*aggregatedRecord{
"Test_aggregator_metric1": {
"cluster 1|CRITICAL": &aggregatedRecord{
labels: prometheus.Labels{
"Cluster": testData[0]["Cluster"],
"Severity": testData[0]["Severity"]},
total: 2,
}},
"Test_aggregator_metric2": {
"ns 1": &aggregatedRecord{
labels: prometheus.Labels{
"Namespace": testData[0]["Namespace"]},
total: 2,
}},
}, a.result)

a.count(testFinding(1))

assert.Equal(t, map[MetricName]map[aggregationKey]*aggregatedRecord{
"Test_aggregator_metric1": {
"cluster 1|CRITICAL": &aggregatedRecord{
labels: prometheus.Labels{
"Cluster": testData[0]["Cluster"],
"Severity": testData[0]["Severity"]},
total: 2,
},
"cluster 2|HIGH": &aggregatedRecord{
labels: prometheus.Labels{
"Cluster": testData[1]["Cluster"],
"Severity": testData[1]["Severity"]},
total: 1,
},
},
"Test_aggregator_metric2": {
"ns 1": &aggregatedRecord{
labels: prometheus.Labels{"Namespace": testData[0]["Namespace"]},
total: 2,
},
"ns 2": &aggregatedRecord{
labels: prometheus.Labels{"Namespace": testData[1]["Namespace"]},
total: 1,
},
}}, a.result)

a.count(testFinding(1))

assert.Equal(t, map[MetricName]map[aggregationKey]*aggregatedRecord{
"Test_aggregator_metric1": {
"cluster 1|CRITICAL": &aggregatedRecord{
labels: prometheus.Labels{"Cluster": testData[0]["Cluster"], "Severity": testData[0]["Severity"]},
total: 2,
},
"cluster 2|HIGH": &aggregatedRecord{
labels: prometheus.Labels{"Cluster": testData[1]["Cluster"], "Severity": testData[1]["Severity"]},
total: 2,
},
},
"Test_aggregator_metric2": {
"ns 1": &aggregatedRecord{
labels: prometheus.Labels{"Namespace": testData[0]["Namespace"]},
total: 2,
},
"ns 2": &aggregatedRecord{
labels: prometheus.Labels{"Namespace": testData[1]["Namespace"]},
total: 2,
},
},
}, a.result)
}

func Test_makeAggregationKey(t *testing.T) {
testMetric := map[Label]string{
"Cluster": "value",
"IsFixable": "false",
}
getter := func(label Label) string {
return testMetric[label]
}
key, labels := makeAggregationKey(
[]Label{"Cluster", "IsFixable"},
getter,
testLabelOrder)

assert.Equal(t, aggregationKey("value|false"), key)
assert.Equal(t, prometheus.Labels{
"Cluster": "value",
"IsFixable": "false",
}, labels)

}

func Test_collectMatchingLabels(t *testing.T) {
i := 0
for range collectMatchingLabels([]Label{"label1", "label2", "label3"},
func(l Label) string {
i++
return "value"
}) {
break
}
assert.Equal(t, 1, i)
}
38 changes: 38 additions & 0 deletions central/metrics/aggregator/common/configuration.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package common

import (
"slices"
)

type Label string // Prometheus label.
type MetricName string // Prometheus metric name.

// MetricsConfiguration is the parsed aggregation configuration.
type MetricsConfiguration map[MetricName][]Label

func (mcfg MetricsConfiguration) hasAnyLabelOf(labels []Label) bool {
for _, configLabels := range mcfg {
for _, label := range configLabels {
if slices.Contains(labels, label) {
return true
}
}
}
return false
}

func (mcfg MetricsConfiguration) diffLabels(another MetricsConfiguration) (toAdd []MetricName, toDelete []MetricName, changed []MetricName) {
for metric, labels := range mcfg {
if anotherLabels, ok := another[metric]; !ok {
toDelete = append(toDelete, metric)
} else if !slices.Equal(labels, anotherLabels) {
changed = append(changed, metric)
}

Check warning on line 30 in central/metrics/aggregator/common/configuration.go

View check run for this annotation

Codecov / codecov/patch

central/metrics/aggregator/common/configuration.go#L29-L30

Added lines #L29 - L30 were not covered by tests
}
for metric := range another {
if _, ok := mcfg[metric]; !ok {
toAdd = append(toAdd, metric)
}
}
return toAdd, toDelete, changed
}
Loading
Loading