Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions docker-compose-testing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,29 @@ services:
--host 0.0.0.0 \
--port \
1199
native_test_mockserver:
networks:
testing-network:
aliases:
- native-test.test
image: stackql/testlib
build:
context: .
dockerfile: testLib.Dockerfile
ports:
- "1070:1070/tcp"
environment:
IS_DOCKER: "${IS_DOCKER:-false}"
command:
- bash
- -c
- |
flask \
--app=/opt/testlib/test/python/stackql_test_tooling/flask/native_test/app \
run \
--host 0.0.0.0 \
--port \
1070

networks:
testing-network:
Expand Down
3 changes: 2 additions & 1 deletion get-app/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { CLOUD_SHELL_SCRIPTS, type CloudShellScript } from "./cloud-shell-scripts";

const GITHUB_REPO = "stackql/stackql";
const RELEASE_BASE = `https://github.com/${GITHUB_REPO}/releases/latest/download`;
// const RELEASE_BASE = `https://github.com/${GITHUB_REPO}/releases/latest/download`;
const RELEASE_BASE = "https://releases.stackql.io/stackql/latest";
const RELEASES_URL = `https://github.com/${GITHUB_REPO}/releases/latest`;
const DOCS_URL = "https://stackql.io/docs";

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.10.1
github.com/stackql/any-sdk v0.5.3-alpha05
github.com/stackql/any-sdk v0.5.3-alpha10
github.com/stackql/go-suffix-map v0.0.1-alpha01
github.com/stackql/psql-wire v0.1.2-beta01
github.com/stackql/stackql-parser v0.0.16-alpha02
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
github.com/stackql/any-sdk v0.5.3-alpha05 h1:Be5d0k5o1fLotY06ToKseNxp/JTFp2P4LpSxspedGw4=
github.com/stackql/any-sdk v0.5.3-alpha05/go.mod h1:yGAr3eEtY6UpFMq8fF5BTvZhdqYGpyS90LP6KY5AKZs=
github.com/stackql/any-sdk v0.5.3-alpha10 h1:FlumRS0hICuUqztUsvRCdHljSLA+aTqrU0Y4Z7DF8WQ=
github.com/stackql/any-sdk v0.5.3-alpha10/go.mod h1:yGAr3eEtY6UpFMq8fF5BTvZhdqYGpyS90LP6KY5AKZs=
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
github.com/stackql/psql-wire v0.1.2-beta01 h1:+0xeLLam3nbI1iv84pWJYGYw5VO5S/+eICRbdu6+dj8=
Expand Down
8 changes: 5 additions & 3 deletions internal/stackql/astvisit/fragment_rewriting.go
Original file line number Diff line number Diff line change
Expand Up @@ -1045,11 +1045,13 @@ func (v *standardFragmentRewriteAstVisitor) Visit(node sqlparser.SQLNode) error
if node == nil {
return nil
}
buf.AstPrintf(node, " limit ")
// Emit standard `LIMIT <count> OFFSET <offset>` (ANSI / postgres / sqlite) rather
// than the MySQL-only `LIMIT <offset>, <count>` form, so the rewritten backend
// query is valid on the postgres SQL backend as well as sqlite.
buf.AstPrintf(node, " limit %v", node.Rowcount)
if node.Offset != nil {
buf.AstPrintf(node, "%v, ", node.Offset)
buf.AstPrintf(node, " offset %v", node.Offset)
}
buf.AstPrintf(node, "%v", node.Rowcount)
v.rewrittenQuery = buf.String()

case sqlparser.Values:
Expand Down
25 changes: 25 additions & 0 deletions internal/stackql/dependencyplanner/dependencyplanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/stackql/stackql/internal/stackql/parserutil"
"github.com/stackql/stackql/internal/stackql/primitivebuilder"
"github.com/stackql/stackql/internal/stackql/primitivecomposer"
"github.com/stackql/stackql/internal/stackql/pushdown"
"github.com/stackql/stackql/internal/stackql/sqlrewrite"
"github.com/stackql/stackql/internal/stackql/sqlstream"
"github.com/stackql/stackql/internal/stackql/tableinsertioncontainer"
Expand Down Expand Up @@ -646,6 +647,7 @@ func (dp *standardDependencyPlanner) orchestrate(
annotationCtx.GetTableMeta(),
)
bldrInput.SetIsAwait(false) // returning hardcoded to false for now
setAcquirePushdownPlan(bldrInput, annotationCtx, dp.sqlStatement)
builder = primitivebuilder.NewSingleSelectAcquire(
dp.primitiveComposer.GetGraphHolder(),
dp.handlerCtx,
Expand All @@ -664,6 +666,29 @@ func (dp *standardDependencyPlanner) orchestrate(
return idx, err
}

// setAcquirePushdownPlan records the analysis-phase query-option push-down decision for a
// SELECT acquire. The SELECT's WHERE / ORDER BY / LIMIT / OFFSET / projection / COUNT become
// a neutral PushdownIntent stored on the table metadata; the HTTP request-prepare phase hands
// it to the preparator, which (inside any-sdk) translates it to the method's dialect and sets
// the request query. A pushable LIMIT is also carried on the plan as the GraphQL page-size
// bound. Both are carried out downstream - the executor stays protocol-agnostic - and the
// client-side WHERE / projection / LIMIT are unchanged, so push-down is only ever an
// optimisation.
func setAcquirePushdownPlan(
bldrInput builder_input.BuilderInput,
annotationCtx taxonomy.AnnotationCtx,
stmt *sqlparser.Select,
) {
tableMeta := annotationCtx.GetTableMeta()
method, _ := tableMeta.GetMethod()
if intent, ok := pushdown.ComputeIntent(stmt, method); ok {
tableMeta.SetPushdownIntent(intent)
}
if limit, ok := pushdown.SelectLimit(stmt); ok {
bldrInput.SetPushdownLimit(limit)
}
}

//nolint:gocognit,funlen // live with it
func (dp *standardDependencyPlanner) processAcquire(
sqlNode sqlparser.SQLNode,
Expand Down
12 changes: 11 additions & 1 deletion internal/stackql/drm/drm_cfg.go
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,17 @@ func (dc *staticDRMConfig) generateVarArgs(
psArgs := cp.GetArgs()
if len(psArgs) > 0 && cp.GetCtx().GetCtrlColumnRepeats() > 0 {
for _, col := range cp.GetCtx().GetNonControlColumns() {
va, ok := psArgs[col.GetName()]
// Extract response values by the key any-sdk used for the row payload, which is the
// schema property key. That equals GetWireName when the wire name carries the case
// difference (e.g. snake_case_aliases: display cidr_block over wire cidrBlock) and
// GetName when the display name does (e.g. an xml:name override: display
// AvailabilityZone over wire availabilityZone). Try the wire name first, then fall
// back to the display name, so both cases resolve their value instead of NULL. For an
// undecorated column GetWireName == GetName, so this is unchanged.
va, ok := psArgs[col.GetWireName()]
if !ok {
va, ok = psArgs[col.GetName()]
}
if !ok {
varArgs = append(varArgs, nil)
continue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ type BuilderInput interface {
SetIsUndo(isUndo bool)
SetRequiredDataRequestKey(key string)
GetRequiredDataRequestKey() (string, bool)
SetPushdownLimit(limit int)
GetPushdownLimit() (int, bool)
SetDependencyNode(dependencyNode primitivegraph.PrimitiveNode)
SetParserNode(node sqlparser.SQLNode)
SetParamMap(paramMap map[int]map[string]interface{})
Expand Down Expand Up @@ -86,6 +88,8 @@ type builderInput struct {
tableInsertionContainer tableinsertioncontainer.TableInsertionContainer
insertCtx drm.PreparedStatementCtx
requiredDataRequestKey string
pushdownLimit int
pushdownLimitSet bool
}

func NewBuilderInput(
Expand All @@ -106,6 +110,17 @@ func (bi *builderInput) GetRequiredDataRequestKey() (string, bool) {
return bi.requiredDataRequestKey, bi.requiredDataRequestKey != ""
}

// SetPushdownLimit records the LIMIT the analysis phase resolved as pushable (used by the
// GraphQL acquire path to bound the page size); the acquire only carries it out.
func (bi *builderInput) SetPushdownLimit(limit int) {
bi.pushdownLimit = limit
bi.pushdownLimitSet = true
}

func (bi *builderInput) GetPushdownLimit() (int, bool) {
return bi.pushdownLimit, bi.pushdownLimitSet
}

func (bi *builderInput) SetRequiredDataRequestKey(key string) {
bi.requiredDataRequestKey = key
}
Expand Down Expand Up @@ -300,5 +315,7 @@ func (bi *builderInput) Clone() BuilderInput {
isReturning: bi.isReturning,
insertCtx: bi.insertCtx,
requiredDataRequestKey: bi.requiredDataRequestKey,
pushdownLimit: bi.pushdownLimit,
pushdownLimitSet: bi.pushdownLimitSet,
}
}
12 changes: 12 additions & 0 deletions internal/stackql/primitivebuilder/graphql_single_select_acquire.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,12 @@ func (ss *GraphQLSingleSelectAcquire) Build() error {

for _, reqCtx := range httpArmoury.GetRequestParams() {
req := reqCtx.GetRequest()
// Emit the GraphQL wire request + raw response when --http.log.enabled is set
// (alpha08 ContextWithHTTPLogger), mirroring the REST acquire path.
if ss.handlerCtx.GetRuntimeContext().HTTPLogEnabled {
req = req.WithContext(
graphql.ContextWithHTTPLogger(req.Context(), ss.handlerCtx.GetOutErrFile()))
}
housekeepingDone := false
cc := formulation.NewAnySdkClientConfigurator(
ss.handlerCtx.GetRuntimeContext(), prov.GetProviderString(),
Expand All @@ -118,6 +124,12 @@ func (ss *GraphQLSingleSelectAcquire) Build() error {
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
}
// Carry out the push-down plan: a SQL LIMIT N resolved during the analysis phase
// becomes the GraphQL `limit` query variable, so a provider query template
// referencing {{ .limit }} can bound the page size.
if limit, hasLimit := ss.bldrInput.GetPushdownLimit(); hasLimit {
paramMap["limit"] = limit
}
cursorJsonPath, ok := gql.GetCursorJSONPath()
if !ok {
return internaldto.NewErroneousExecutorOutput(fmt.Errorf("cannot perform graphql action without cursor json path"))
Expand Down
35 changes: 32 additions & 3 deletions internal/stackql/primitivecomposer/primitive_composer.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,30 @@ func (pb *standardPrimitiveComposer) GetChildren() []PrimitiveComposer {
return pb.children
}

// withNativeCasingAliases adds each native-casing (snake_case) parameter alias to the same
// required/optional map as its wire parameter, so snake_case SQL keys resolve to the wire
// parameter. The alias and its wire key share an Addressable whose GetName is the wire name.
// any-sdk owns the casing translation; for methods without request native casing the
// supplied allWithCasing map equals the wire param set and this is a no-op.
func withNativeCasingAliases(
reqParams, optParams, allWithCasing map[string]formulation.Addressable,
) (map[string]formulation.Addressable, map[string]formulation.Addressable) {
for aliasKey, addr := range allWithCasing {
if _, isWire := reqParams[aliasKey]; isWire {
continue
}
if _, isWire := optParams[aliasKey]; isWire {
continue
}
if _, req := reqParams[addr.GetName()]; req {
reqParams[aliasKey] = addr
} else if _, opt := optParams[addr.GetName()]; opt {
optParams[aliasKey] = addr
}
}
return reqParams, optParams
}

//nolint:gocognit // acceptable for now
func (pb *standardPrimitiveComposer) AssignParameters() (internaldto.TableParameterCollection, error) {
requiredParameters := suffix.NewParameterSuffixMap()
Expand All @@ -281,9 +305,14 @@ func (pb *standardPrimitiveComposer) AssignParameters() (internaldto.TableParame
tblOptParams = assignedParams.GetOptionalParams().GetAll()
remainingReqParams = assignedParams.GetRemainingRequiredParams().GetAll()
} else {
// These methods need to incorporate request body parameters
reqParams = tb.GetRequiredParameters()
tblOptParams = tb.GetOptionalParameters()
// These methods need to incorporate request body parameters.
// Register native-casing (snake_case) aliases so snake_case WHERE / INSERT keys
// resolve to their wire parameter (any-sdk owns the casing translation).
reqParams, tblOptParams = withNativeCasingAliases(
tb.GetRequiredParameters(),
tb.GetOptionalParameters(),
tb.GetParametersIncludingNativeCasing(),
)
}
// This method needs to incorporate request body parameters

Expand Down
Loading
Loading