-
Notifications
You must be signed in to change notification settings - Fork 174
ROX-32714: Enable feature flag by default (Still updating failing tests. Do not review) #18539
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
charmik-redhat
wants to merge
15
commits into
master
Choose a base branch
from
ROX-32714/enable-flatten-image-data-flag
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
f0a887c
Enable feature flag by default
charmik-redhat 9e5e408
Fix a bunch of more tests
charmik-redhat 1bb8f94
Fix some more
charmik-redhat fac441d
More test fixes
charmik-redhat f60b13f
Fix tests
charmik-redhat d2616cc
Fix UI tests
charmik-redhat cbc57ea
Fix groovy tests and one more view test
charmik-redhat a5ad802
Remove unnecessary line deletes
charmik-redhat 5284708
Fix UI test
charmik-redhat 7c9373f
More fixes
charmik-redhat c257f6b
Add migration and address comments
charmik-redhat 7d7d644
Fix style
charmik-redhat baed312
Fix tests that broke when claude tried to address coderabbit's comments
charmik-redhat adf04e7
Retry getImages in UpgradesTest when FlattenImageData is enabled
charmik-redhat ff4bb4f
Remove benchmark results file
charmik-redhat File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
migrator/migrations/m_223_to_m_224_populate_deployment_containers_imageidv2/migration.go
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
79 changes: 79 additions & 0 deletions
79
...igrations/m_223_to_m_224_populate_deployment_containers_imageidv2/migration_bench_test.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| //go:build sql_integration | ||
|
|
||
| package m223tom224 | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "testing" | ||
|
|
||
| "github.com/stackrox/rox/migrator/migrations/m_223_to_m_224_populate_deployment_containers_imageidv2/schema" | ||
| pghelper "github.com/stackrox/rox/migrator/migrations/postgreshelper" | ||
| "github.com/stackrox/rox/migrator/types" | ||
| "github.com/stackrox/rox/pkg/postgres/pgutils" | ||
| "github.com/stackrox/rox/pkg/sac" | ||
| "github.com/stackrox/rox/pkg/uuid" | ||
| ) | ||
|
|
||
| func BenchmarkMigration(b *testing.B) { | ||
| ctx := sac.WithAllAccess(context.Background()) | ||
| db := pghelper.ForT(b, false) | ||
| pgutils.CreateTableFromModel(ctx, db.GetGormDB(), schema.CreateTableDeploymentsStmt) | ||
|
|
||
| const numDeployments = 100 | ||
| const containersPerDeployment = 5 | ||
| totalContainers := numDeployments * containersPerDeployment | ||
|
|
||
| // Insert deployments and containers. | ||
| for i := 0; i < numDeployments; i++ { | ||
| depID := uuid.NewV4().String() | ||
| err := insertIntoDeployments(ctx, db, &schema.Deployments{ | ||
| ID: depID, | ||
| Name: fmt.Sprintf("dep-%d", i), | ||
| Type: "Deployment", | ||
| Namespace: "default", | ||
| }) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| for j := 0; j < containersPerDeployment; j++ { | ||
| sql := "INSERT INTO deployments_containers (image_name_fullname, image_id, deployments_id, idx) VALUES ($1, $2, $3, $4)" | ||
| fullName := fmt.Sprintf("registry.example.com/image-%d:%d@sha256:%064x", i, j, i*containersPerDeployment+j) | ||
| imageID := fmt.Sprintf("sha256:%064x", i*containersPerDeployment+j) | ||
| _, err := db.Exec(ctx, sql, fullName, imageID, depID, j) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| b.ResetTimer() | ||
| for n := 0; n < b.N; n++ { | ||
| // Reset image_idv2 so the migration has work to do each iteration. | ||
| _, err := db.Exec(ctx, "UPDATE deployments_containers SET image_idv2 = ''") | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
|
|
||
| batchSize = 5000 | ||
| dbs := &types.Databases{ | ||
| GormDB: db.GetGormDB(), | ||
| PostgresDB: db.DB, | ||
| DBCtx: ctx, | ||
| } | ||
| if err := migrate(dbs); err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| } | ||
| b.StopTimer() | ||
|
|
||
| // Verify all rows were populated. | ||
| var count int | ||
| err := db.QueryRow(ctx, "SELECT COUNT(*) FROM deployments_containers WHERE image_idv2 != '' AND image_idv2 IS NOT NULL").Scan(&count) | ||
| if err != nil { | ||
| b.Fatal(err) | ||
| } | ||
| if count != totalContainers { | ||
| b.Fatalf("expected %d containers with image_idv2, got %d", totalContainers, count) | ||
| } | ||
| } |
94 changes: 94 additions & 0 deletions
94
...ator/migrations/m_223_to_m_224_populate_deployment_containers_imageidv2/migration_impl.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| package m223tom224 | ||
|
|
||
| import ( | ||
| "github.com/hashicorp/go-multierror" | ||
| "github.com/jackc/pgx/v5" | ||
| "github.com/stackrox/rox/migrator/migrations/m_223_to_m_224_populate_deployment_containers_imageidv2/schema" | ||
| "github.com/stackrox/rox/migrator/types" | ||
| "github.com/stackrox/rox/pkg/logging" | ||
| "github.com/stackrox/rox/pkg/postgres" | ||
| "github.com/stackrox/rox/pkg/postgres/pgutils" | ||
| "github.com/stackrox/rox/pkg/uuid" | ||
| ) | ||
|
|
||
| var ( | ||
| log = logging.LoggerForModule() | ||
| batchSize = 5000 | ||
| ) | ||
|
|
||
| func migrate(database *types.Databases) error { | ||
| // Use databases.DBCtx to take advantage of the transaction wrapping present in the migration initiator | ||
| pgutils.CreateTableFromModel(database.DBCtx, database.GormDB, schema.CreateTableDeploymentsStmt) | ||
| log.Infof("Batch size is %d", batchSize) | ||
|
|
||
| db := database.PostgresDB | ||
|
|
||
| conn, err := db.Acquire(database.DBCtx) | ||
| defer conn.Release() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| updatedRows := 0 | ||
| for { | ||
| batch := pgx.Batch{} | ||
| // This will continue looping through the containers until there are no more containers that need to have their | ||
| // image_idv2 field populated, in batches up to batchSize | ||
| getStmt := `SELECT image_name_fullname, image_id FROM deployments_containers WHERE image_id is not null AND image_id != '' AND image_name_fullname is not null AND image_name_fullname != '' AND (image_idv2 is null OR image_idv2 = '') LIMIT $1` | ||
| rows, err := db.Query(database.DBCtx, getStmt, batchSize) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer rows.Close() | ||
|
|
||
| containers, err := readRows(rows) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for _, container := range containers { | ||
| updateStmt := `UPDATE deployments_containers SET image_idv2 = $1 WHERE image_name_fullname = $2 AND image_id = $3` | ||
| imageIdV2 := uuid.NewV5FromNonUUIDs(container.ImageNameFullName, container.ImageID).String() | ||
| batch.Queue(updateStmt, imageIdV2, container.ImageNameFullName, container.ImageID) | ||
| } | ||
| batchResults := conn.SendBatch(database.DBCtx, &batch) | ||
| var result *multierror.Error | ||
| for i := 0; i < batch.Len(); i++ { | ||
| _, err = batchResults.Exec() | ||
| result = multierror.Append(result, err) | ||
| if err == nil { | ||
| updatedRows += 1 | ||
| } | ||
| } | ||
| if err = batchResults.Close(); err != nil { | ||
| return err | ||
| } | ||
| if err = result.ErrorOrNil(); err != nil { | ||
| return err | ||
| } | ||
| if len(containers) != batchSize { | ||
| log.Infof("Populated the image_idv2 field in deployment containers. %d rows updated.", updatedRows) | ||
| return nil | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func readRows(rows *postgres.Rows) ([]*schema.DeploymentsContainers, error) { | ||
| var containers []*schema.DeploymentsContainers | ||
|
|
||
| for rows.Next() { | ||
| var imageName string | ||
| var imageId string | ||
|
|
||
| if err := rows.Scan(&imageName, &imageId); err != nil { | ||
| log.Errorf("Error scanning row: %v", err) | ||
| } | ||
|
|
||
| container := &schema.DeploymentsContainers{ | ||
| ImageID: imageId, | ||
| ImageNameFullName: imageName, | ||
| } | ||
| containers = append(containers, container) | ||
| } | ||
|
|
||
| log.Debugf("Read returned %d containers", len(containers)) | ||
| return containers, rows.Err() | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Scan errors are silently swallowed — risk of infinite loop and data corruption.
rows.Scanerrors are logged but not returned, and the (now empty) container is still appended on line 89. Two consequences:ImageNameFullName/ImageIDgets a bogus UPDATE queued that matches nothing (the SELECT filter excludes empty values), so the row is silently skipped.len(containers) == batchSizestill holds (we kept appending), the outer loop re-queries the same rows, scan-fails again, and loops forever.Return the scan error immediately, or at least
continuewithout appending.🛡️ Proposed fix
if err := rows.Scan(&imageName, &imageId); err != nil { - log.Errorf("Error scanning row: %v", err) + return nil, err }📝 Committable suggestion
🤖 Prompt for AI Agents