Skip to content

ROX-13326: Use UUID columns for roles, permissionsets and simpleaccessscopes (PermissionSet and SimpleAccessScope identifiers) in postgres mode. #3725

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rhybrillou merged 10 commits into from
Nov 17, 2022
Merged

ROX-13326: Use UUID columns for roles, permissionsets and simpleaccessscopes (PermissionSet and SimpleAccessScope identifiers) in postgres mode. #3725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d9abd0a
ROX-13326: Use UUID columns for roles, permissionsets and simpleacces…
dashrews78 Nov 3, 2022
53ded04
Add full migration test
rhybrillou Nov 14, 2022
ae6397c
Style fixes
rhybrillou Nov 14, 2022
c8d861e
Extend test to validate UUID access scope and permission set identifi…
rhybrillou Nov 15, 2022
85c6d50
Fix migrator build issues
rhybrillou Nov 15, 2022
1b76b83
Refine migrator test
rhybrillou Nov 15, 2022
49588ae
Type management fixes
rhybrillou Nov 15, 2022
1d580ea
Fix test
rhybrillou Nov 15, 2022
a4c6ba5
ROX-13326: Access control UUID support in the UI (#3818)
rhybrillou Nov 17, 2022
78f731f
Address review comment
rhybrillou Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions central/role/accessscope_ids.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package role

// postgres ID for default sccess scopes
// Postgres IDs for access scopes
// The values are UUIDs taken in descending order from ffffffff-ffff-fff4-f5ff-ffffffffffff
// Next ID: ffffffff-ffff-fff4-f5ff-fffffffffffd
const (
denyAllAccessScopeID = "ffffffff-ffff-fff4-f5ff-fffffffffffe"
unrestrictedAccessScopeID = "ffffffff-ffff-fff4-f5ff-ffffffffffff"
denyAllAccessScopeID = "ffffffff-ffff-fff4-f5ff-fffffffffffe"
)
2 changes: 1 addition & 1 deletion central/role/datastore/permissionset_ids.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package datastore

// The postgresID field is there for UUID support of permission set ID fields in postgres mode.
// Postgres IDs for permission sets
// The values are UUIDs taken in descending order from ffffffff-ffff-fff4-f5ff-ffffffffffff
// Next ID: ffffffff-ffff-fff4-f5ff-fffffffffff6
const (
Expand Down
2 changes: 1 addition & 1 deletion central/role/datastore/singleton.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func Singleton() DataStore {

type roleAttributes struct {
idSuffix string
postgresID string
postgresID string // postgresID should be populated with valid UUID values.
description string
resourceWithAccess []permissions.ResourceWithAccess
}
Expand Down
4 changes: 3 additions & 1 deletion central/role/store/permissionset/postgres/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
package postgres

//go:generate pg-table-bindings-wrapper --type=storage.PermissionSet --migration-seq 37 --migrate-from rocksdb
//go:generate pg-table-bindings-wrapper --type=storage.PermissionSet
// To regenerate migration, add:
Copy link
Copy Markdown
Contributor

@mtodor mtodor Nov 8, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand what this change will do. Can you explain it to me?

Copy link
Copy Markdown
Contributor Author

@rhybrillou rhybrillou Nov 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the --migration-seq command line parameter allows the generation of the n_... data migration files.

As the migration requires to process permissionset, role and simpleaccessscope together in order to reconcile the non-uuid identifiers, the current migration code generation cannot be used.

As an example, the Admin role uses the admin permission set and the unrestricted access scope. The migration will have to replace admin with the same permission set identifier value in both permission_sets and the roles tables, as well as the unrestricted with the same access scope identifier value in both simple_access_scopes and roles tables.

The generated code for permissionset and role is moved to the simpleaccessscopes migration file.

// --migration-seq 37 --migrate-from rocksdb
4 changes: 3 additions & 1 deletion central/role/store/permissionset/rocksdb/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
package rocksdb

//go:generate rocksdb-bindings-wrapper --type=PermissionSet --bucket=permission_sets --cache --uniq-key-func GetName() --migration-seq 37 --migrate-to permission_sets
//go:generate rocksdb-bindings-wrapper --type=PermissionSet --bucket=permission_sets --cache --uniq-key-func GetName()
// To regenerate migration, add:
// --migration-seq 37 --migrate-to permission_sets
4 changes: 3 additions & 1 deletion central/role/store/role/postgres/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
package postgres

//go:generate pg-table-bindings-wrapper --type=storage.Role --migration-seq 46 --migrate-from rocksdb
//go:generate pg-table-bindings-wrapper --type=storage.Role
// To regenerate migration, add:
// --migration-seq 46 --migrate-from rocksdb
4 changes: 3 additions & 1 deletion central/role/store/role/rocksdb/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
package rocksdb

//go:generate rocksdb-bindings-wrapper --type=Role --bucket=roles --cache --key-func GetName() --migration-seq 46 --migrate-to roles
//go:generate rocksdb-bindings-wrapper --type=Role --bucket=roles --cache --key-func GetName()
// To regenerate migration, add:
// --migration-seq 46 --migrate-to roles
4 changes: 3 additions & 1 deletion central/role/store/simpleaccessscope/postgres/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
package postgres

//go:generate pg-table-bindings-wrapper --type=storage.SimpleAccessScope --migration-seq 52 --migrate-from rocksdb
//go:generate pg-table-bindings-wrapper --type=storage.SimpleAccessScope
// To regenerate migration, add:
// --migration-seq 52 --migrate-from rocksdb
4 changes: 3 additions & 1 deletion central/role/store/simpleaccessscope/rocksdb/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
package rocksdb

//go:generate rocksdb-bindings-wrapper --type=SimpleAccessScope --bucket=simple_access_scopes --cache --uniq-key-func GetName() --migration-seq 52 --migrate-to simple_access_scopes
//go:generate rocksdb-bindings-wrapper --type=SimpleAccessScope --bucket=simple_access_scopes --cache --uniq-key-func GetName()
// To regenerate migration, add:
// --migration-seq 52 --migrate-to simple_access_scopes
60 changes: 6 additions & 54 deletions migrator/migrations/n_37_to_n_38_postgres_permission_sets/migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,76 +1,28 @@
// Code generated by pg-bindings generator. DO NOT EDIT.
package n37ton38

import (
"context"
// Code generation from pg-bindings generator disabled. To re-enable, check the gen.go file in
// central/role/store/permissionset/postgres

"github.com/jackc/pgx/v4/pgxpool"
"github.com/pkg/errors"
import (
"github.com/stackrox/rox/generated/storage"
"github.com/stackrox/rox/migrator/migrations"
"github.com/stackrox/rox/migrator/migrations/loghelper"
legacy "github.com/stackrox/rox/migrator/migrations/n_37_to_n_38_postgres_permission_sets/legacy"
pgStore "github.com/stackrox/rox/migrator/migrations/n_37_to_n_38_postgres_permission_sets/postgres"
"github.com/stackrox/rox/migrator/types"
pkgMigrations "github.com/stackrox/rox/pkg/migrations"
pkgSchema "github.com/stackrox/rox/pkg/postgres/schema"
"github.com/stackrox/rox/pkg/sac"
"gorm.io/gorm"
)

var (
migration = types.Migration{
StartingSeqNum: pkgMigrations.CurrentDBVersionSeqNumWithoutPostgres() + 37,
VersionAfter: &storage.Version{SeqNum: int32(pkgMigrations.CurrentDBVersionSeqNumWithoutPostgres()) + 38},
Run: func(databases *types.Databases) error {
legacyStore, err := legacy.New(databases.PkgRocksDB)
if err != nil {
return err
}
if err := move(databases.GormDB, databases.PostgresDB, legacyStore); err != nil {
return errors.Wrap(err,
"moving permission_sets from rocksdb to postgres")
}
// The data migration code was moved to the simpleaccessscope migrator.
// The goal is to be able to convert the IDs that do not parse as UUIDs to proper UUID values,
// and still be able to convert the references in the roles table.
return nil
},
}
batchSize = 10000
schema = pkgSchema.PermissionSetsSchema
log = loghelper.LogWrapper{}
)

func move(gormDB *gorm.DB, postgresDB *pgxpool.Pool, legacyStore legacy.Store) error {
ctx := sac.WithAllAccess(context.Background())
store := pgStore.New(postgresDB)
pkgSchema.ApplySchemaForTable(context.Background(), gormDB, schema.Table)
var permissionSets []*storage.PermissionSet
err := walk(ctx, legacyStore, func(obj *storage.PermissionSet) error {
permissionSets = append(permissionSets, obj)
if len(permissionSets) == batchSize {
if err := store.UpsertMany(ctx, permissionSets); err != nil {
log.WriteToStderrf("failed to persist permission_sets to store %v", err)
return err
}
permissionSets = permissionSets[:0]
}
return nil
})
if err != nil {
return err
}
if len(permissionSets) > 0 {
if err = store.UpsertMany(ctx, permissionSets); err != nil {
log.WriteToStderrf("failed to persist permission_sets to store %v", err)
return err
}
}
return nil
}

func walk(ctx context.Context, s legacy.Store, fn func(obj *storage.PermissionSet) error) error {
return s.Walk(ctx, fn)
}

func init() {
migrations.MustRegisterMigration(migration)
}
40 changes: 3 additions & 37 deletions migrator/migrations/n_37_to_n_38_postgres_permission_sets/migration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,18 @@
// Code generated by pg-bindings generator. DO NOT EDIT.

//go:build sql_integration

package n37ton38

// Code generation from pg-bindings generator disabled. To re-enable, check the gen.go file in
// central/role/store/permissionset/postgres

import (
"context"
"testing"

"github.com/stackrox/rox/generated/storage"
legacy "github.com/stackrox/rox/migrator/migrations/n_37_to_n_38_postgres_permission_sets/legacy"
pgStore "github.com/stackrox/rox/migrator/migrations/n_37_to_n_38_postgres_permission_sets/postgres"
pghelper "github.com/stackrox/rox/migrator/migrations/postgreshelper"

"github.com/stackrox/rox/pkg/env"
"github.com/stackrox/rox/pkg/rocksdb"
"github.com/stackrox/rox/pkg/sac"
"github.com/stackrox/rox/pkg/testutils"
"github.com/stackrox/rox/pkg/testutils/rocksdbtest"
"github.com/stretchr/testify/suite"
)
Expand Down Expand Up @@ -56,33 +52,3 @@ func (s *postgresMigrationSuite) TearDownTest() {
rocksdbtest.TearDownRocksDB(s.legacyDB)
s.postgresDB.Teardown(s.T())
}

func (s *postgresMigrationSuite) TestPermissionSetMigration() {
newStore := pgStore.New(s.postgresDB.Pool)
legacyStore, err := legacy.New(s.legacyDB)
s.NoError(err)

// Prepare data and write to legacy DB
var permissionSets []*storage.PermissionSet
for i := 0; i < 200; i++ {
permissionSet := &storage.PermissionSet{}
s.NoError(testutils.FullInit(permissionSet, testutils.UniqueInitializer(), testutils.JSONFieldsFilter))
permissionSets = append(permissionSets, permissionSet)
}

s.NoError(legacyStore.UpsertMany(s.ctx, permissionSets))

// Move
s.NoError(move(s.postgresDB.GetGormDB(), s.postgresDB.Pool, legacyStore))

// Verify
count, err := newStore.Count(s.ctx)
s.NoError(err)
s.Equal(len(permissionSets), count)
for _, permissionSet := range permissionSets {
fetched, exists, err := newStore.Get(s.ctx, permissionSet.GetId())
s.NoError(err)
s.True(exists)
s.Equal(permissionSet, fetched)
}
}
56 changes: 7 additions & 49 deletions migrator/migrations/n_46_to_n_47_postgres_roles/migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
// Code generated by pg-bindings generator. DO NOT EDIT.
package n46ton47

import (
"context"
// Code generation from pg-bindings generator disabled. To re-enable, check the gen.go file in
// central/role/store/role/postgres

import (
"github.com/jackc/pgx/v4/pgxpool"
"github.com/pkg/errors"
"github.com/stackrox/rox/generated/storage"
"github.com/stackrox/rox/migrator/migrations"
"github.com/stackrox/rox/migrator/migrations/loghelper"
legacy "github.com/stackrox/rox/migrator/migrations/n_46_to_n_47_postgres_roles/legacy"
pgStore "github.com/stackrox/rox/migrator/migrations/n_46_to_n_47_postgres_roles/postgres"
"github.com/stackrox/rox/migrator/types"
pkgMigrations "github.com/stackrox/rox/pkg/migrations"
pkgSchema "github.com/stackrox/rox/pkg/postgres/schema"
"github.com/stackrox/rox/pkg/sac"
"gorm.io/gorm"
)

Expand All @@ -23,54 +17,18 @@ var (
StartingSeqNum: pkgMigrations.CurrentDBVersionSeqNumWithoutPostgres() + 46,
VersionAfter: &storage.Version{SeqNum: int32(pkgMigrations.CurrentDBVersionSeqNumWithoutPostgres()) + 47},
Run: func(databases *types.Databases) error {
legacyStore, err := legacy.New(databases.PkgRocksDB)
if err != nil {
return err
}
if err := move(databases.GormDB, databases.PostgresDB, legacyStore); err != nil {
return errors.Wrap(err,
"moving roles from rocksdb to postgres")
}
// The data migration code was moved to the simpleaccessscope migrator.
// The goal is to be able to convert the IDs that do not parse as UUIDs to proper UUID values,
// and still be able to convert the references in the roles table.
return nil
},
}
batchSize = 10000
schema = pkgSchema.RolesSchema
log = loghelper.LogWrapper{}
)

func move(gormDB *gorm.DB, postgresDB *pgxpool.Pool, legacyStore legacy.Store) error {
ctx := sac.WithAllAccess(context.Background())
store := pgStore.New(postgresDB)
pkgSchema.ApplySchemaForTable(context.Background(), gormDB, schema.Table)
var roles []*storage.Role
err := walk(ctx, legacyStore, func(obj *storage.Role) error {
roles = append(roles, obj)
if len(roles) == batchSize {
if err := store.UpsertMany(ctx, roles); err != nil {
log.WriteToStderrf("failed to persist roles to store %v", err)
return err
}
roles = roles[:0]
}
return nil
})
if err != nil {
return err
}
if len(roles) > 0 {
if err = store.UpsertMany(ctx, roles); err != nil {
log.WriteToStderrf("failed to persist roles to store %v", err)
return err
}
}
func move(gormDB *gorm.DB, postgresDB *pgxpool.Pool) error {
return nil
}

func walk(ctx context.Context, s legacy.Store, fn func(obj *storage.Role) error) error {
return s.Walk(ctx, fn)
}

func init() {
migrations.MustRegisterMigration(migration)
}
40 changes: 3 additions & 37 deletions migrator/migrations/n_46_to_n_47_postgres_roles/migration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,18 @@
// Code generated by pg-bindings generator. DO NOT EDIT.

//go:build sql_integration

package n46ton47

// Code generation from pg-bindings generator disabled. To re-enable, check the gen.go file in
// central/role/store/role/postgres

import (
"context"
"testing"

"github.com/stackrox/rox/generated/storage"
legacy "github.com/stackrox/rox/migrator/migrations/n_46_to_n_47_postgres_roles/legacy"
pgStore "github.com/stackrox/rox/migrator/migrations/n_46_to_n_47_postgres_roles/postgres"
pghelper "github.com/stackrox/rox/migrator/migrations/postgreshelper"

"github.com/stackrox/rox/pkg/env"
"github.com/stackrox/rox/pkg/rocksdb"
"github.com/stackrox/rox/pkg/sac"
"github.com/stackrox/rox/pkg/testutils"
"github.com/stackrox/rox/pkg/testutils/rocksdbtest"
"github.com/stretchr/testify/suite"
)
Expand Down Expand Up @@ -56,33 +52,3 @@ func (s *postgresMigrationSuite) TearDownTest() {
rocksdbtest.TearDownRocksDB(s.legacyDB)
s.postgresDB.Teardown(s.T())
}

func (s *postgresMigrationSuite) TestRoleMigration() {
newStore := pgStore.New(s.postgresDB.Pool)
legacyStore, err := legacy.New(s.legacyDB)
s.NoError(err)

// Prepare data and write to legacy DB
var roles []*storage.Role
for i := 0; i < 200; i++ {
role := &storage.Role{}
s.NoError(testutils.FullInit(role, testutils.UniqueInitializer(), testutils.JSONFieldsFilter))
roles = append(roles, role)
}

s.NoError(legacyStore.UpsertMany(s.ctx, roles))

// Move
s.NoError(move(s.postgresDB.GetGormDB(), s.postgresDB.Pool, legacyStore))

// Verify
count, err := newStore.Count(s.ctx)
s.NoError(err)
s.Equal(len(roles), count)
for _, role := range roles {
fetched, exists, err := newStore.Get(s.ctx, role.GetName())
s.NoError(err)
s.True(exists)
s.Equal(role, fetched)
}
}
6 changes: 3 additions & 3 deletions ..._permission_sets/legacy/rocksdb_plugin.go → ...es/legacypermissionsets/rocksdb_plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
// Code generated by rocksdb-bindings generator. DO NOT EDIT.
package legacy
package legacypermissionsets

import (
"context"
Expand All @@ -15,6 +14,7 @@ var (
bucket = []byte("permission_sets")
)

// Store is the interface for interactions with the database storage
type Store interface {
UpsertMany(ctx context.Context, objs []*storage.PermissionSet) error
Walk(ctx context.Context, fn func(obj *storage.PermissionSet) error) error
Expand All @@ -38,7 +38,7 @@ func uniqKeyFunc(msg proto.Message) []byte {
// New returns a new Store instance using the provided rocksdb instance.
func New(db *rocksdb.RocksDB) (Store, error) {
baseCRUD := generic.NewUniqueKeyCRUD(db, bucket, keyFunc, alloc, uniqKeyFunc, false)
return &storeImpl{crud: baseCRUD}, nil
return &storeImpl{crud: baseCRUD}, nil
}

// UpsertMany batches objects into the DB
Expand Down
Loading