Skip to content

ROX-32848: ACK-based retry logic for VM index reports #18489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
vikin91 wants to merge 2 commits into from
Closed

ROX-32848: ACK-based retry logic for VM index reports #18489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
204bc80
Empty
vikin91 Jan 14, 2026
8cba653
Draft new VM index report retry logic
vikin91 Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion compliance/cmd/compliance/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func main() {
defer cancel()
umhNodeInv := handler.NewUnconfirmedMessageHandler(ctx, "node-inventory", env.NodeScanningAckDeadlineBase.DurationSetting())
umhNodeIndex := handler.NewUnconfirmedMessageHandler(ctx, "node-index", env.NodeScanningAckDeadlineBase.DurationSetting())
c := compliance.NewComplianceApp(np, scanner, cachedNodeIndexer, umhNodeInv, umhNodeIndex)
umhVMIndex := handler.NewUnconfirmedMessageHandler(ctx, "vm-index", env.NodeScanningAckDeadlineBase.DurationSetting())
c := compliance.NewComplianceApp(np, scanner, cachedNodeIndexer, umhNodeInv, umhNodeIndex, umhVMIndex)
c.Start()
}
139 changes: 117 additions & 22 deletions compliance/compliance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"os"
"os/signal"
"strconv"
"syscall"
"time"

Expand All @@ -14,6 +15,7 @@ import (
cmetrics "github.com/stackrox/rox/compliance/collection/metrics"
"github.com/stackrox/rox/compliance/node"
"github.com/stackrox/rox/compliance/virtualmachines/relay"
vmmetrics "github.com/stackrox/rox/compliance/virtualmachines/relay/metrics"
"github.com/stackrox/rox/compliance/virtualmachines/relay/sender"
"github.com/stackrox/rox/compliance/virtualmachines/relay/stream"
v4 "github.com/stackrox/rox/generated/internalapi/scanner/v4"
Expand All @@ -35,25 +37,33 @@ import (

var log = logging.LoggerForModule()

const (
// nodeResourceID is the resource ID used for node scanning UMH.
// Compliance handles exactly one node, so a single constant suffices.
nodeResourceID = "this-node"
)

// Compliance represents the Compliance app
type Compliance struct {
nodeNameProvider node.NodeNameProvider
nodeScanner node.NodeScanner
nodeIndexer node.NodeIndexer
umhNodeInventory node.UnconfirmedMessageHandler
umhNodeIndex node.UnconfirmedMessageHandler
umhVMIndex node.UnconfirmedMessageHandler
cache *sensor.MsgFromCompliance
}

// NewComplianceApp constructs the Compliance app object
func NewComplianceApp(nnp node.NodeNameProvider, scanner node.NodeScanner, nodeIndexer node.NodeIndexer,
umhNodeInv, umhNodeIndex node.UnconfirmedMessageHandler) *Compliance {
umhNodeInv, umhNodeIndex, umhVMIndex node.UnconfirmedMessageHandler) *Compliance {
return &Compliance{
nodeNameProvider: nnp,
nodeScanner: scanner,
nodeIndexer: nodeIndexer,
umhNodeInventory: umhNodeInv,
umhNodeIndex: umhNodeIndex,
umhVMIndex: umhVMIndex,
cache: nil,
}
}
Expand Down Expand Up @@ -139,7 +149,19 @@ func (c *Compliance) Start() {
sensorClient := sensor.NewVirtualMachineIndexReportServiceClient(conn)
reportSender := sender.New(sensorClient)

vmRelay := relay.New(reportStream, reportSender)
maxPerMinuteStr := env.VMRelayMaxReportsPerMinute.Setting()
maxPerMinute, err := strconv.ParseFloat(maxPerMinuteStr, 64)
if err != nil {
log.Panicf("Failed to parse ROX_VM_RELAY_MAX_REPORTS_PER_MINUTE '%s' as float: %v", maxPerMinuteStr, err)
}

vmRelay := relay.New(
reportStream,
reportSender,
c.umhVMIndex,
maxPerMinute,
env.VMRelayStaleAckThreshold.DurationSetting(),
)
if err := vmRelay.Run(ctx); err != nil {
log.Errorf("Error running virtual machine relay: %v", err)
}
Expand Down Expand Up @@ -179,12 +201,16 @@ func (c *Compliance) manageNodeInventoryScanLoop(ctx context.Context) <-chan *se
select {
case <-ctx.Done():
return
case _, ok := <-c.umhNodeInventory.RetryCommand():
case resourceID, ok := <-c.umhNodeInventory.RetryCommand():
if !ok {
log.Info("UMH retry channel for node inventory closed; stopping scan loop")
return
}
if c.cache == nil {
log.Debug("Requested to retry but cache is empty. Resetting scan timer.")
log.Debugf("Requested to retry %s but cache is empty. Resetting scan timer.", resourceID)
cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheMiss, cmetrics.ScannerVersionV2)
t.Reset(time.Second)
} else if ok {
} else {
nodeInventoriesC <- c.cache
cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheHit, cmetrics.ScannerVersionV2)
}
Expand Down Expand Up @@ -215,12 +241,16 @@ func (c *Compliance) manageNodeIndexScanLoop(ctx context.Context) <-chan *sensor
select {
case <-ctx.Done():
return
case _, ok := <-c.umhNodeIndex.RetryCommand():
case resourceID, ok := <-c.umhNodeIndex.RetryCommand():
if !ok {
log.Info("UMH retry channel for node index closed; stopping scan loop")
return
}
if c.cache == nil {
log.Debug("Requested to retry but cache is empty. Resetting scan timer.")
log.Debugf("Requested to retry %s but cache is empty. Resetting scan timer.", resourceID)
cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheMiss, cmetrics.ScannerVersionV4)
t.Reset(time.Second)
} else if ok {
} else {
nodeIndexesC <- c.cache
cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionResendingCacheHit, cmetrics.ScannerVersionV4)
}
Expand Down Expand Up @@ -249,7 +279,7 @@ func (c *Compliance) runNodeInventoryScan(ctx context.Context) *sensor.MsgFromCo
}
cmetrics.ObserveNodeInventoryScan(msg.GetNodeInventory())
cmetrics.ObserveNodePackageReportTransmissions(nodeName, cmetrics.InventoryTransmissionScan, cmetrics.ScannerVersionV2)
c.umhNodeInventory.ObserveSending()
c.umhNodeInventory.ObserveSending(nodeResourceID)
c.cache = msg.CloneVT()
return msg
}
Expand All @@ -266,7 +296,7 @@ func (c *Compliance) runNodeIndex(ctx context.Context) *sensor.MsgFromCompliance
log.Errorf("Error creating node index: %v", err)
return nil
}
c.umhNodeIndex.ObserveSending()
c.umhNodeIndex.ObserveSending(nodeResourceID)
cmetrics.ObserveNodeIndexReport(report, nodeName)
msg := c.createIndexMsg(report, nodeName)
cmetrics.ObserveReportProtobufMessage(msg, cmetrics.ScannerVersionV4)
Expand Down Expand Up @@ -343,43 +373,108 @@ func (c *Compliance) runRecv(ctx context.Context, client sensor.ComplianceServic
}
}
case *sensor.MsgToCompliance_Ack:
// Legacy NodeInventoryACK from Sensor 4.9 and earlier
switch t.Ack.GetAction() {
case sensor.MsgToCompliance_NodeInventoryACK_ACK:
switch t.Ack.GetMessageType() {
case sensor.MsgToCompliance_NodeInventoryACK_NodeInventory:
c.umhNodeInventory.HandleACK()
c.umhNodeInventory.HandleACK(nodeResourceID)
case sensor.MsgToCompliance_NodeInventoryACK_NodeIndexer:
c.umhNodeIndex.HandleACK()
c.umhNodeIndex.HandleACK(nodeResourceID)
default:
log.Errorf("Unknown ACK Type: %s", t.Ack.GetMessageType())
}
case sensor.MsgToCompliance_NodeInventoryACK_NACK:
switch t.Ack.GetMessageType() {
case sensor.MsgToCompliance_NodeInventoryACK_NodeInventory:
c.umhNodeInventory.HandleNACK()
c.umhNodeInventory.HandleNACK(nodeResourceID)
case sensor.MsgToCompliance_NodeInventoryACK_NodeIndexer:
c.umhNodeIndex.HandleNACK()
c.umhNodeIndex.HandleNACK(nodeResourceID)
default:
log.Errorf("Unknown ACK Type: %s", t.Ack.GetMessageType())
}
default:
log.Errorf("Unknown ACK Action: %s", t.Ack.GetAction())
}
case *sensor.MsgToCompliance_ComplianceAck:
complianceAck := t.ComplianceAck
log.Debugf("Received ComplianceACK: type=%s, action=%s, resource_id=%s, reason=%s",
complianceAck.GetMessageType(),
complianceAck.GetAction(),
complianceAck.GetResourceId(),
complianceAck.GetReason(),
)
// TODO: Handle ComplianceACK message from Sensor/Central 4.10.
// New ComplianceACK from Sensor 4.10+
c.handleComplianceACK(t.ComplianceAck)
default:
utils.Should(errors.Errorf("Unhandled msg type: %T", t))
}
}
}

// handleComplianceACK handles the new ComplianceACK message from Sensor 4.10+.
// This is the generic ACK/NACK message that replaces the legacy NodeInventoryACK.
func (c *Compliance) handleComplianceACK(ack *sensor.MsgToCompliance_ComplianceACK) {
if ack == nil {
log.Error("Received nil ComplianceACK")
return
}

log.Debugf("Received ComplianceACK: type=%s, action=%s, resource_id=%s, reason=%s",
ack.GetMessageType(), ack.GetAction(), ack.GetResourceId(), ack.GetReason())

switch ack.GetMessageType() {
case sensor.MsgToCompliance_ComplianceACK_NODE_INVENTORY:
c.handleNodeInventoryACK(ack.GetAction(), ack.GetReason())
case sensor.MsgToCompliance_ComplianceACK_NODE_INDEX_REPORT:
c.handleNodeIndexACK(ack.GetAction(), ack.GetReason())
case sensor.MsgToCompliance_ComplianceACK_VM_INDEX_REPORT:
c.handleVMIndexACK(ack.GetResourceId(), ack.GetAction(), ack.GetReason())
default:
log.Errorf("Unknown ComplianceACK message type: %s", ack.GetMessageType())
}
}

// handleNodeInventoryACK handles ACK/NACK for node inventory messages.
func (c *Compliance) handleNodeInventoryACK(action sensor.MsgToCompliance_ComplianceACK_Action, reason string) {
switch action {
case sensor.MsgToCompliance_ComplianceACK_ACK:
c.umhNodeInventory.HandleACK(nodeResourceID)
case sensor.MsgToCompliance_ComplianceACK_NACK:
if reason != "" {
log.Infof("Node inventory NACK received: %s", reason)
}
c.umhNodeInventory.HandleNACK(nodeResourceID)
default:
log.Errorf("Unknown ComplianceACK action for node inventory: %s", action)
}
}

// handleNodeIndexACK handles ACK/NACK for node index report messages.
func (c *Compliance) handleNodeIndexACK(action sensor.MsgToCompliance_ComplianceACK_Action, reason string) {
switch action {
case sensor.MsgToCompliance_ComplianceACK_ACK:
c.umhNodeIndex.HandleACK(nodeResourceID)
case sensor.MsgToCompliance_ComplianceACK_NACK:
if reason != "" {
log.Infof("Node index NACK received: %s", reason)
}
c.umhNodeIndex.HandleNACK(nodeResourceID)
default:
log.Errorf("Unknown ComplianceACK action for node index: %s", action)
}
}

// handleVMIndexACK handles ACK/NACK for VM index report messages.
func (c *Compliance) handleVMIndexACK(resourceID string, action sensor.MsgToCompliance_ComplianceACK_Action, reason string) {
switch action {
case sensor.MsgToCompliance_ComplianceACK_ACK:
vmmetrics.VMIndexACKsFromSensor.WithLabelValues("ACK").Inc()
c.umhVMIndex.HandleACK(resourceID)
case sensor.MsgToCompliance_ComplianceACK_NACK:
vmmetrics.VMIndexACKsFromSensor.WithLabelValues("NACK").Inc()
if reason != "" {
log.Infof("VM index NACK received for %s: %s", resourceID, reason)
}
c.umhVMIndex.HandleNACK(resourceID)
default:
log.Errorf("Unknown ComplianceACK action for VM index: %s", action)
}
}

func (c *Compliance) startAuditLogCollection(ctx context.Context, client sensor.ComplianceService_CommunicateClient, request *sensor.MsgToCompliance_AuditLogCollectionRequest_StartRequest) auditlog.Reader {
if request.GetCollectStartState() == nil {
log.Infof("Starting audit log reader on node %s in cluster %s with no saved state", c.nodeNameProvider.GetNodeName(), request.GetClusterId())
Expand Down
Loading
Loading