Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,8 @@ jobs:
BUILDBRANCH: ${{github.ref}}
BUILDPLATFORM: ${{runner.os}}
BUILDPATCHVERSION: ${{github.run_number}}
CGO_ENABLED: 1
CGO_LDFLAGS: '-static'
run: |
source version.txt
BUILDMAJORVERSION=$MajorVersion
Expand Down
2 changes: 1 addition & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"select ipCidrRange, sum(5) cc from google.container.\"projects.aggregated.usableSubnetworks\" where projectsId = 'testing-project' group by ipCidrRange having sum(5) >= 5 order by ipCidrRange desc;",
"show providers;",
"select u.name, om.login, u.two_factor_authentication AS is_two_fa_enabled from github.orgs.members om inner join github.users.users u on om.login = u.login AND u.username = om.login where om.org = 'dummyorg' order by u.name desc;",
"select * from stackql_providers;",
"select * from stackql_repositories;",
"describe sumologic.users.users;",
"select id from sumologic.collectors.collectors order by id desc;",
"select VolumeId, Encrypted, Size from aws.ec2.volumes where region = 'ap-southeast-1' order by VolumeId asc;"
Expand Down
162 changes: 162 additions & 0 deletions internal/stackql/astanalysis/earlyanalysis/first_passes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package earlyanalysis

import (
"github.com/stackql/stackql/internal/stackql/astvisit"
"github.com/stackql/stackql/internal/stackql/handler"
"github.com/stackql/stackql/internal/stackql/internaldto"
"github.com/stackql/stackql/internal/stackql/logging"
"github.com/stackql/stackql/internal/stackql/parse"
"github.com/stackql/stackql/internal/stackql/planbuilderinput"
"vitess.io/vitess/go/vt/sqlparser"
)

type InstructionType int

const (
StandardInstruction InstructionType = iota
CachedInstruction
InternallyRoutableInstruction
DummiedPGInstruction
NopInstruction
)

type Analyzer interface {
Analyze(handlerCtx handler.HandlerContext, tcc internaldto.TxnControlCounters) error
}

type InitialPassesScreener interface {
GetInstructionType() InstructionType
GetPlanBuilderInput() planbuilderinput.PlanBuilderInput
GetStatementType() sqlparser.StatementType
IsCacheExemptMaterialDetected() bool
}

type InitialPassesScreenerAnalyzer interface {
Analyzer
InitialPassesScreener
}

var (
_ InitialPassesScreenerAnalyzer = &standardInitialPasses{}
)

func NewEarlyScreenerAnalyzer() (InitialPassesScreenerAnalyzer, error) {
return &standardInitialPasses{}, nil
}

type standardInitialPasses struct {
instructionType InstructionType
isCacheExemptMaterialDetected bool
planBuilderInput planbuilderinput.PlanBuilderInput
result *sqlparser.RewriteASTResult
}

func (sp *standardInitialPasses) GetInstructionType() InstructionType {
return sp.instructionType
}

func (sp *standardInitialPasses) GetPlanBuilderInput() planbuilderinput.PlanBuilderInput {
return sp.planBuilderInput
}

func (sp *standardInitialPasses) GetStatementType() sqlparser.StatementType {
return sqlparser.ASTToStatementType(sp.result.AST)
}

func (sp *standardInitialPasses) IsCacheExemptMaterialDetected() bool {
return sp.isCacheExemptMaterialDetected
}

func (sp *standardInitialPasses) Analyze(handlerCtx handler.HandlerContext, tcc internaldto.TxnControlCounters) error {
return sp.initialPasses(handlerCtx, tcc)
}

func (sp *standardInitialPasses) initialPasses(handlerCtx handler.HandlerContext, tcc internaldto.TxnControlCounters) error {
statement, err := parse.ParseQuery(handlerCtx.GetQuery())
if err != nil {
return err
}
result, err := sqlparser.RewriteAST(statement)
sp.result = result
if err != nil {
return err
}

// Before analysing AST, see if we can pass stright to SQL backend
opType, ok := handlerCtx.GetDBMSInternalRouter().CanRoute(result.AST)
if ok {
sp.instructionType = InternallyRoutableInstruction
logging.GetLogger().Debugf("%v", opType)
pbi, err := planbuilderinput.NewPlanBuilderInput(handlerCtx, result.AST, nil, nil, nil, nil, nil, tcc.Clone())
if err != nil {
return err
}
sp.planBuilderInput = pbi
return nil
}

// First pass AST analysis; extract provider strings for auth.
provStrSlice, isCacheExemptMaterialDetected := astvisit.ExtractProviderStringsAndDetectCacheExceptMaterial(result.AST, handlerCtx.GetSQLDialect(), handlerCtx.GetASTFormatter(), handlerCtx.GetNamespaceCollection())
sp.isCacheExemptMaterialDetected = isCacheExemptMaterialDetected
for _, p := range provStrSlice {
_, err := handlerCtx.GetProvider(p)
if err != nil {
return err
}
}

ast := result.AST

// Second pass AST analysis; extract provider strings for auth.
// Extracts:
// - parser objects representing tables.
// - mapping of string aliases to tables.
tVis := astvisit.NewTableExtractAstVisitor()
tVis.Visit(ast)

// Third pass AST analysis.
// Accepts slice of parser table objects
// extracted from previous analysis.
// Extracts:
// - Col Refs; mapping columnar objects to tables.
// - Alias Map; mapping the "TableName" objects
// defining aliases to table objects.
aVis := astvisit.NewTableAliasAstVisitor(tVis.GetTables())
aVis.Visit(ast)

// Fourth pass AST analysis.
// Extracts:
// - Columnar parameters with null values.
// Useful for method matching.
// Especially for "Insert" queries.
tpv := astvisit.NewPlaceholderParamAstVisitor("", false)
tpv.Visit(ast)

pbi, err := planbuilderinput.NewPlanBuilderInput(handlerCtx, ast, tVis.GetTables(), aVis.GetAliasedColumns(), tVis.GetAliasMap(), aVis.GetColRefs(), tpv.GetParameters(), tcc.Clone())
if err != nil {
return err
}

if sel, ok := planbuilderinput.IsPGSetupQuery(pbi); ok {
if sel != nil {
sp.instructionType = DummiedPGInstruction
pbi, err := planbuilderinput.NewPlanBuilderInput(handlerCtx, result.AST, nil, nil, nil, nil, nil, tcc.Clone())
if err != nil {
return err
}
sp.planBuilderInput = pbi
return nil
} else {
sp.instructionType = NopInstruction
pbi, err := planbuilderinput.NewPlanBuilderInput(handlerCtx, nil, nil, nil, nil, nil, nil, tcc.Clone())
if err != nil {
return err
}
sp.planBuilderInput = pbi
return nil
}
}
sp.instructionType = StandardInstruction
sp.planBuilderInput = pbi
return nil
}
59 changes: 59 additions & 0 deletions internal/stackql/astindirect/indirect.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package astindirect

import (
"fmt"

"github.com/stackql/stackql/internal/stackql/parse"

"github.com/stackql/stackql/internal/stackql/internaldto"
"vitess.io/vitess/go/vt/sqlparser"
)

var (
_ Indirect = &view{}
)

type IndirectType int

const (
ViewType IndirectType = iota
SubqueryType
CTEType
)

func NewViewIndirect(viewDTO internaldto.ViewDTO) (Indirect, error) {
rv := &view{
viewDTO: viewDTO,
}
return rv, nil
}

type Indirect interface {
GetSelectAST() (sqlparser.SelectStatement, error)
GetType() IndirectType
}

type view struct {
viewDTO internaldto.ViewDTO
}

func (v *view) GetType() IndirectType {
return ViewType
}

func (v *view) getAST() (sqlparser.Statement, error) {
return parse.ParseQuery(v.viewDTO.GetRawQuery())
}

func (v *view) GetSelectAST() (sqlparser.SelectStatement, error) {
parseResult, err := v.getAST()
if err != nil {
return nil, err
}
switch pr := parseResult.(type) {
case sqlparser.SelectStatement:
return pr, nil
default:
return nil, fmt.Errorf("view of type '%T' not yet supported", pr)
}
}
1 change: 1 addition & 0 deletions internal/stackql/dataflow/collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type DataFlowUnit interface {
iDataFlowUnit()
}

// DataFlowCollection is the DAG representing
type DataFlowCollection interface {
AddOrUpdateEdge(source DataFlowVertex, dest DataFlowVertex, comparisonExpr *sqlparser.ComparisonExpr, sourceExpr sqlparser.Expr, destColumn *sqlparser.ColName) error
AddVertex(v DataFlowVertex)
Expand Down
2 changes: 1 addition & 1 deletion internal/stackql/dependencyplanner/dependencyplanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (dp *standardDependencyPlanner) Plan() error {
}
tableExpr := unit.GetTableExpr()
annotation := unit.GetAnnotation()
if annotation.IsView() {
if _, isView := annotation.GetView(); isView {
return fmt.Errorf("error in dependency planning: views in progress")
}
dp.annMap[tableExpr] = annotation
Expand Down
70 changes: 70 additions & 0 deletions internal/stackql/internaldto/db_table.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package internaldto

import (
"github.com/stackql/stackql/internal/stackql/constants"
)

var (
_ DBTable = &standardDBTable{}
)

type DBTable interface {
GetBaseName() string
GetDiscoveryID() int
GetHeirarchyIdentifiers() HeirarchyIdentifiers
IsAnalytics() bool
GetName() string
GetNameStump() string
}

type standardDBTable struct {
name string
nameStump string
baseName string
discoveryID int
hIDs HeirarchyIdentifiers
namespace string
}

func NewDBTable(name string, nameStump string, baseName string, discoveryID int, hIDs HeirarchyIdentifiers) DBTable {
return newDBTable(name, nameStump, baseName, discoveryID, hIDs, "")
}

func NewDBTableAnalytics(name string, discoveryID int, hIDs HeirarchyIdentifiers) DBTable {
return newDBTable(name, name, name, discoveryID, hIDs, constants.AnalyticsPrefix)
}

func newDBTable(name string, nameStump string, baseName string, discoveryID int, hIDs HeirarchyIdentifiers, namespace string) DBTable {
return &standardDBTable{
name: name,
nameStump: nameStump,
baseName: baseName,
discoveryID: discoveryID,
hIDs: hIDs,
namespace: namespace,
}
}

func (dbt *standardDBTable) GetName() string {
return dbt.name
}

func (dbt *standardDBTable) GetNameStump() string {
return dbt.nameStump
}

func (dbt *standardDBTable) GetBaseName() string {
return dbt.baseName
}

func (dbt *standardDBTable) GetDiscoveryID() int {
return dbt.discoveryID
}

func (dbt *standardDBTable) GetHeirarchyIdentifiers() HeirarchyIdentifiers {
return dbt.hIDs
}

func (dbt *standardDBTable) IsAnalytics() bool {
return dbt.namespace == constants.AnalyticsPrefix
}
Loading