Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@
"select ipCidrRange, sum(5) cc from google.container.\"projects.aggregated.usableSubnetworks\" where projectsId = 'testing-project' group by ipCidrRange having sum(5) >= 5 order by ipCidrRange desc;",
"show providers;",
"select u.name, om.login, u.two_factor_authentication AS is_two_fa_enabled from github.orgs.members om inner join github.users.users u on om.login = u.login AND u.username = om.login where om.org = 'dummyorg' order by u.name desc;",
"select * from stackql_repositories;",
"select id, name, url from stackql_repositories;",
"describe sumologic.users.users;",
"select id from sumologic.collectors.collectors order by id desc;",
"select VolumeId, Encrypted, Size from aws.ec2.volumes where region = 'ap-southeast-1' order by VolumeId asc;"
"select VolumeId, Encrypted, Size from aws.ec2.volumes where region = 'ap-southeast-1' order by VolumeId asc;",
"SELECT DISTINCT \"EventTime\", \"Identifier\" from aws.cloud_control.resource_requests where data__ResourceRequestStatusFilter='{}' and region = 'ap-southeast-1' order by \"Identifier\", \"EventTime\";"
],
"default": "show providers;"
},
Expand Down
2 changes: 2 additions & 0 deletions internal/stackql/astanalysis/annotatedast/annotated_ast.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ func (aa *standardAnnotatedAst) GetAST() sqlparser.Statement {

func (aa *standardAnnotatedAst) GetIndirect(node sqlparser.SQLNode) (astindirect.Indirect, bool) {
switch n := node.(type) {
case *sqlparser.AliasedTableExpr:
return aa.GetIndirect(n.Expr)
case sqlparser.TableName:
rv, ok := aa.tableIndirects[n.GetRawVal()]
return rv, ok
Expand Down
1 change: 1 addition & 0 deletions internal/stackql/astanalysis/earlyanalysis/ast_expand.go
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,7 @@ func (v *indirectExpandAstVisitor) Visit(node sqlparser.SQLNode) error {
if err != nil {
return err
}
indirect.SetSelectContext(indirectPrimitiveGenerator.GetPrimitiveComposer().GetSelectPreparedStatementCtx())
v.annotatedAST.SetIndirect(node, indirect)
}
return nil
Expand Down
30 changes: 30 additions & 0 deletions internal/stackql/astindirect/indirect.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,13 @@ func NewViewIndirect(viewDTO internaldto.ViewDTO) (Indirect, error) {

type Indirect interface {
Parse() error
GetColumnByName(name string) (drm.ColumnMetadata, bool)
GetColumns() []drm.ColumnMetadata
GetName() string
GetSelectAST() sqlparser.SelectStatement
GetSelectContext() drm.PreparedStatementCtx
GetType() IndirectType
SetSelectContext(drm.PreparedStatementCtx)
}

type view struct {
Expand All @@ -45,6 +50,31 @@ func (v *view) GetType() IndirectType {
return ViewType
}

func (v *view) GetName() string {
return v.viewDTO.GetName()
}

func (v *view) GetColumns() []drm.ColumnMetadata {
return v.selCtx.GetNonControlColumns()
}

func (v *view) GetColumnByName(name string) (drm.ColumnMetadata, bool) {
for _, col := range v.selCtx.GetNonControlColumns() {
if col.GetName() == name {
return col, true
}
}
return nil, false
}

func (v *view) SetSelectContext(selCtx drm.PreparedStatementCtx) {
v.selCtx = selCtx
}

func (v *view) GetSelectContext() drm.PreparedStatementCtx {
return v.selCtx
}

func (v *view) GetTables() sqlparser.TableExprs {
return nil
}
Expand Down
53 changes: 34 additions & 19 deletions internal/stackql/astvisit/from_rewrite.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ var (

type FromRewriteAstVisitor interface {
sqlparser.SQLAstVisitor
GetIndirectContexts() []drm.PreparedStatementCtx
GetRewrittenQuery() string
}

Expand All @@ -35,6 +36,7 @@ type standardFromRewriteAstVisitor struct {
annotations taxonomy.AnnotationCtxMap
dc drm.DRMConfig
annotatedAST annotatedast.AnnotatedAst
indirectContexts []drm.PreparedStatementCtx
}

func NewFromRewriteAstVisitor(
Expand Down Expand Up @@ -63,6 +65,10 @@ func (v *standardFromRewriteAstVisitor) GetRewrittenQuery() string {
return v.rewrittenQuery
}

func (v *standardFromRewriteAstVisitor) GetIndirectContexts() []drm.PreparedStatementCtx {
return v.indirectContexts
}

func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
buf := sqlparser.NewTrackedBuffer(v.formatter)

Expand Down Expand Up @@ -619,25 +625,34 @@ func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
if !ok {
return fmt.Errorf("cannot find annotated tabulation for table object")
}
switch ex := node.Expr.(type) {
case sqlparser.TableName:
if ex.IsEmpty() {
return nil
}
dbTbl, err := v.dc.GetCurrentTable(anCtx.GetHIDs())
if err != nil {
return err
}
tblStr := dbTbl.GetName()
fqtn, err := v.sqlDialect.GetFullyQualifiedTableName(tblStr)
v.rewrittenQuery = fqtn
if err != nil {
return err
}
default:
err := node.Expr.Accept(v)
if err != nil {
return err
if indirect, isIndirect := anCtx.GetTableMeta().GetIndirect(); isIndirect {
//
alias := indirect.GetName()
templateString := fmt.Sprintf(` ( %%s ) AS %s `, alias)
v.rewrittenQuery = templateString
v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext())

} else {
switch ex := node.Expr.(type) {
case sqlparser.TableName:
if ex.IsEmpty() {
return nil
}
dbTbl, err := v.dc.GetCurrentTable(anCtx.GetHIDs())
if err != nil {
return err
}
tblStr := dbTbl.GetName()
fqtn, err := v.sqlDialect.GetFullyQualifiedTableName(tblStr)
v.rewrittenQuery = fqtn
if err != nil {
return err
}
default:
err := node.Expr.Accept(v)
if err != nil {
return err
}
}
}
exprStr = v.GetRewrittenQuery()
Expand Down
57 changes: 45 additions & 12 deletions internal/stackql/astvisit/query_rewriting.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/stackql/stackql/internal/stackql/internaldto"
"github.com/stackql/stackql/internal/stackql/logging"
"github.com/stackql/stackql/internal/stackql/parserutil"
"github.com/stackql/stackql/internal/stackql/relationaldto"
"github.com/stackql/stackql/internal/stackql/sqlrewrite"
"github.com/stackql/stackql/internal/stackql/tableinsertioncontainer"
"github.com/stackql/stackql/internal/stackql/tablemetadata"
Expand Down Expand Up @@ -43,6 +44,7 @@ type standardQueryRewriteAstVisitor struct {
colRefs parserutil.ColTableMap
columnNames []parserutil.ColumnHandle
columnDescriptors []openapistackql.ColumnDescriptor
relationalColumns []relationaldto.RelationalColumn
tableSlice []tableinsertioncontainer.TableInsertionContainer
namespaceCollection tablenamespace.TableNamespaceCollection
formatter sqlparser.NodeFormatter
Expand All @@ -53,6 +55,8 @@ type standardQueryRewriteAstVisitor struct {
selectSuffix string
// singe threaded, so no mutex protection
anonColCounter int
//
indirectContexts []drm.PreparedStatementCtx
}

func NewQueryRewriteAstVisitor(
Expand Down Expand Up @@ -95,7 +99,14 @@ func (v *standardQueryRewriteAstVisitor) getNextAlias() string {

func (v *standardQueryRewriteAstVisitor) getStarColumns(
tbl tablemetadata.ExtendedTableMetadata,
) ([]openapistackql.ColumnDescriptor, error) {
) ([]relationaldto.RelationalColumn, error) {
// if view, isView := tbl.GetView(); isView {
// return view.GetRawQuery()
// }
if indirect, isIndirect := tbl.GetIndirect(); isIndirect {
rv := v.dc.ColumnsToRelationalColumns(indirect.GetColumns())
return rv, nil
}
schema, _, err := tbl.GetResponseSchemaAndMediaType()
if err != nil {
return nil, err
Expand All @@ -118,13 +129,15 @@ func (v *standardQueryRewriteAstVisitor) getStarColumns(
for _, col := range cols {
columnDescriptors = append(columnDescriptors, openapistackql.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, nil, schema, col.Val))
}
return columnDescriptors, nil
relationalColumns := v.dc.OpenapiColumnsToRelationalColumns(columnDescriptors)
return relationalColumns, nil
}

func (v *standardQueryRewriteAstVisitor) GenerateSelectDML() (drm.PreparedStatementCtx, error) {
relationalColumns := v.relationalColumns
rewriteInput := sqlrewrite.NewStandardSQLRewriteInput(
v.dc,
v.columnDescriptors,
relationalColumns,
v.baseCtrlCounters,
v.selectSuffix,
v.whereExprsStr,
Expand All @@ -133,7 +146,7 @@ func (v *standardQueryRewriteAstVisitor) GenerateSelectDML() (drm.PreparedStatem
v.fromStr,
v.tableSlice,
v.namespaceCollection,
)
).WithIndirectContexts(v.indirectContexts)
return sqlrewrite.GenerateSelectDML(rewriteInput)
}

Expand Down Expand Up @@ -201,6 +214,7 @@ func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
node.From.Accept(fromVis)
v.fromStr = fromVis.GetRewrittenQuery()
}
v.indirectContexts = fromVis.GetIndirectContexts()
}
if node.Where != nil {
node.Where.Accept(v)
Expand Down Expand Up @@ -389,11 +403,11 @@ func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
case *sqlparser.ExecSubquery:
t, ok := v.annotations[node]
if !ok {
return fmt.Errorf("could not infer annotated table")
return fmt.Errorf("exec: could not infer annotated table")
}
dID, ok := v.discoGenIDs[node]
if !ok {
return fmt.Errorf("could not infer discovery generation ID")
return fmt.Errorf("exec: could not infer discovery generation ID")
}
replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID)
node.Exec.MethodName = replacementExpr
Expand Down Expand Up @@ -529,7 +543,7 @@ func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
if err != nil {
return err
}
v.columnDescriptors = append(v.columnDescriptors, cols...)
v.relationalColumns = append(v.relationalColumns, cols...)

case *sqlparser.AliasedExpr:
tbl, tblErr := v.tables.GetTableLoose(node)
Expand All @@ -548,6 +562,20 @@ func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
v.columnNames = append(v.columnNames, col)
cd := openapistackql.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, node, nil, col.Val)
v.columnDescriptors = append(v.columnDescriptors, cd)
v.relationalColumns = append(v.relationalColumns, v.dc.OpenapiColumnsToRelationalColumn(cd))
return nil
}
if indirect, isIndirect := tbl.GetIndirect(); isIndirect {
col, err := parserutil.InferColNameFromExpr(node, v.formatter)
if err != nil {
return err
}
c, ok := indirect.GetColumnByName(col.Name)
if !ok {
return fmt.Errorf("query rewriting for indirection: cannot find col = '%s'", col.Name)
}
rv := v.dc.ColumnToRelationalColumn(c)
v.relationalColumns = append(v.relationalColumns, rv)
return nil
}
schema, err := tbl.GetSelectableObjectSchema()
Expand All @@ -562,6 +590,7 @@ func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
ss, _ := schema.GetProperty(col.Name)
cd := openapistackql.NewColumnDescriptor(col.Alias, col.Name, col.Qualifier, col.DecoratedColumn, node, ss, col.Val)
v.columnDescriptors = append(v.columnDescriptors, cd)
v.relationalColumns = append(v.relationalColumns, v.dc.OpenapiColumnsToRelationalColumn(cd))
if !node.As.IsEmpty() {
}

Expand Down Expand Up @@ -596,12 +625,16 @@ func (v *standardQueryRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
if !ok {
return fmt.Errorf("could not infer annotated table")
}
dID, ok := v.discoGenIDs[node]
if !ok {
return fmt.Errorf("could not infer discovery generation ID")
if _, isIndirect := t.GetTableMeta().GetIndirect(); isIndirect {
// do nothing
} else {
dID, ok := v.discoGenIDs[node]
if !ok {
return fmt.Errorf("could not infer discovery generation ID")
}
replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID)
node.Expr = replacementExpr
}
replacementExpr := v.dc.GetParserTableName(t.GetHIDs(), dID)
node.Expr = replacementExpr
}
}
if node.Partitions != nil {
Expand Down
6 changes: 5 additions & 1 deletion internal/stackql/dependencyplanner/dependencyplanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ func (dp *standardDependencyPlanner) Plan() error {
tableExpr := unit.GetTableExpr()
annotation := unit.GetAnnotation()
if _, isView := annotation.GetView(); isView {

dp.annMap[tableExpr] = annotation

continue
// return fmt.Errorf("error in dependency planning: views in progress")
}
Expand Down Expand Up @@ -375,9 +378,10 @@ func (dp *standardDependencyPlanner) generateSelectDML(e dataflow.DataFlowEdge,
if alias != "" {
tableName = fmt.Sprintf("%s AS %s", tableName, alias)
}
relationalColumns := dp.handlerCtx.GetDrmConfig().OpenapiColumnsToRelationalColumns(columnDescriptors)
rewriteInput := sqlrewrite.NewStandardSQLRewriteInput(
dp.handlerCtx.GetDrmConfig(),
columnDescriptors,
relationalColumns,
tcc,
"",
"",
Expand Down
Loading