Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.10.1
github.com/stackql/any-sdk v0.5.3-alpha01
github.com/stackql/any-sdk v0.5.3-alpha05
github.com/stackql/go-suffix-map v0.0.1-alpha01
github.com/stackql/psql-wire v0.1.2-beta01
github.com/stackql/stackql-parser v0.0.16-alpha02
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
github.com/stackql/any-sdk v0.5.3-alpha01 h1:z3Y1afNhcR4Nz/geJJr9Pcb+OnCUmI35UL+/gOv5oKU=
github.com/stackql/any-sdk v0.5.3-alpha01/go.mod h1:yGAr3eEtY6UpFMq8fF5BTvZhdqYGpyS90LP6KY5AKZs=
github.com/stackql/any-sdk v0.5.3-alpha05 h1:Be5d0k5o1fLotY06ToKseNxp/JTFp2P4LpSxspedGw4=
github.com/stackql/any-sdk v0.5.3-alpha05/go.mod h1:yGAr3eEtY6UpFMq8fF5BTvZhdqYGpyS90LP6KY5AKZs=
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
github.com/stackql/psql-wire v0.1.2-beta01 h1:+0xeLLam3nbI1iv84pWJYGYw5VO5S/+eICRbdu6+dj8=
Expand Down
98 changes: 73 additions & 25 deletions internal/stackql/primitivebuilder/graphql_single_select_acquire.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,49 +154,36 @@ func (ss *GraphQLSingleSelectAcquire) Build() error {
ss.drmCfg.ExtractObjectFromSQLRows(r, nonControlColumns, ss.stream)
return internaldto.NewEmptyExecutorOutput()
}
graphQLReader, err := graphql.NewStandardGQLReader(
transformType, transformBody := extractGraphQLResponseTransform(ss.tableMeta)
cursorCfg := buildGraphQLCursorConfig(gql, cursorJsonPath)
graphQLReader, err := graphql.NewStandardGQLReaderFull(
client,
req,
ss.handlerCtx.GetRuntimeContext().HTTPPageLimit,
gql.GetQuery(),
paramMap,
"",
responseJsonPath,
cursorJsonPath,
cursorCfg,
transformType,
transformBody,
)
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
}
for {
response, err := graphQLReader.Read()
response, readErr := graphQLReader.Read()
ss.handlerCtx.LogHTTPResponseMap(response)
if len(response) > 0 {
if !housekeepingDone && ss.insertPreparedStatementCtx != nil {
_, err = ss.handlerCtx.GetSQLEngine().Exec(ss.insertPreparedStatementCtx.GetGCHousekeepingQueries())
ss.insertionContainer.SetTableTxnCounters(tableName, ss.insertPreparedStatementCtx.GetGCCtrlCtrs())
housekeepingDone = true
}
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
}
err = ss.stream.Write(response)
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
}
for _, item := range response {
// TODO: handle request encoding
r, err := ss.drmCfg.ExecuteInsertDML(ss.handlerCtx.GetSQLEngine(), ss.insertPreparedStatementCtx, item, "")
logging.GetLogger().Infoln(fmt.Sprintf("insert result = %v, error = %v", r, err))
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
}
if processErr := ss.processGraphQLPage(response, tableName, &housekeepingDone); processErr != nil {
return internaldto.NewErroneousExecutorOutput(processErr)
}
}
if errors.Is(err, io.EOF) {
if errors.Is(readErr, io.EOF) {
break
}
if err != nil {
return internaldto.NewErroneousExecutorOutput(err)
if readErr != nil {
return internaldto.NewErroneousExecutorOutput(readErr)
}
}
}
Expand All @@ -218,3 +205,64 @@ func (ss *GraphQLSingleSelectAcquire) Build() error {

return nil
}

func (ss *GraphQLSingleSelectAcquire) processGraphQLPage(
response []map[string]interface{},
tableName string,
housekeepingDone *bool,
) error {
if !*housekeepingDone && ss.insertPreparedStatementCtx != nil {
_, hkErr := ss.handlerCtx.GetSQLEngine().Exec(ss.insertPreparedStatementCtx.GetGCHousekeepingQueries())
//nolint:errcheck // pre-existing behaviour: housekeeping counter set is best-effort
ss.insertionContainer.SetTableTxnCounters(tableName, ss.insertPreparedStatementCtx.GetGCCtrlCtrs())
*housekeepingDone = true
if hkErr != nil {
return hkErr
}
}
if writeErr := ss.stream.Write(response); writeErr != nil {
return writeErr
}
for _, item := range response {
// TODO: handle request encoding
r, insertErr := ss.drmCfg.ExecuteInsertDML(ss.handlerCtx.GetSQLEngine(), ss.insertPreparedStatementCtx, item, "")
logging.GetLogger().Infoln(fmt.Sprintf("insert result = %v, error = %v", r, insertErr))
if insertErr != nil {
return insertErr
}
}
return nil
}

func extractGraphQLResponseTransform(tableMeta tablemetadata.ExtendedTableMetadata) (string, string) {
op, err := tableMeta.GetMethod()
if err != nil || op == nil {
return "", ""
}
er, ok := op.GetResponse()
if !ok || er == nil {
return "", ""
}
t, ok := er.GetTransform()
if !ok || t == nil {
return "", ""
}
return t.GetType(), t.GetBody()
}

func buildGraphQLCursorConfig(gql formulation.GraphQL, cursorJSONPath string) graphql.CursorConfig {
cfg := graphql.CursorConfig{JSONPath: cursorJSONPath}
if strategy, ok := gql.GetCursorStrategy(); ok && strategy != "" {
cfg.Strategy = graphql.CursorStrategy(strategy)
}
if format, ok := gql.GetCursorFormat(); ok && format != "" {
cfg.FormatTemplate = format
}
if terminator, ok := gql.GetCursorTerminateOnJSONPath(); ok && terminator != "" {
cfg.TerminateOnJSONPath = terminator
}
if pageSize, ok := gql.GetCursorPageSize(); ok && pageSize > 0 {
cfg.PageSize = pageSize
}
return cfg
}
17 changes: 17 additions & 0 deletions test/registry/src/trevorblades/v0.1.0/provider.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
id: trevorblades
name: trevorblades
version: v0.1.0
providerServices:
geo:
description: Public Countries GraphQL API at countries.trevorblades.com. Used to exercise the any-sdk GraphQL response.transform path end-to-end against a live endpoint.
id: geo:v0.1.0
name: geo
preferred: true
service:
$ref: trevorblades/v0.1.0/services/geo.yaml
title: Countries GraphQL
version: v0.1.0
openapi: 3.0.3
config:
auth:
type: "null_auth"
89 changes: 89 additions & 0 deletions test/registry/src/trevorblades/v0.1.0/services/geo.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
openapi: 3.0.3
info:
title: Countries GraphQL
version: v0.1.0
description: |
Live public GraphQL endpoint at https://countries.trevorblades.com.
Used in stackql's integration-traffic-lights suite to prove the any-sdk
GraphQL response.transform path works end-to-end. The transform flattens
each continent's nested countries array into a top-level country_count
column so the SELECT can return scalar columns without JSON_EXTRACT.
servers:
- url: https://countries.trevorblades.com
paths:
/graphql:
post:
x-stackQL-graphQL:
url: https://countries.trevorblades.com/graphql
httpVerb: POST
responseSelection:
jsonPath: '$.data.continents[*]'
cursor:
jsonPath: '$.data.continents[*].__no_cursor'
query: >
query {
continents {
code
name
countries {
code
}
}
}
responses:
'200':
description: Response
content:
application/json:
schema:
type: object
properties:
data:
type: object
properties:
continents:
type: array
items:
type: object
properties:
code:
type: string
name:
type: string
country_count:
type: integer
components:
x-stackQL-resources:
continents:
id: trevorblades.geo.continents
name: continents
title: Continents
methods:
list:
operation:
$ref: '#/paths/~1graphql/post'
response:
mediaType: application/json
openAPIDocKey: '200'
objectKey: '$.data.continents[*]'
transform:
type: golang_template_json_v0.3.0
body: |
{
"data": {
"continents": [
{{- $first := true -}}
{{- range $i, $continent := (index . "data" "continents") }}
{{- if not $first }},{{ end }}{{- $first = false }}
{
"code": {{ toJson (index $continent "code") }},
"name": {{ toJson (index $continent "name") }},
"country_count": {{ len (index $continent "countries") }}
}
{{- end }}
]
}
}
sqlVerbs:
select:
- $ref: '#/components/x-stackQL-resources/continents/methods/list'
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,27 @@ AWS S3 Bucket ABAC Works
... stderr=${CURDIR}/tmp/AWS-S3-Bucket-Objects-List-stderr.tmp
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} stackql\-trial\-bucket\-02

Trevorblades Continents GraphQL Response Transform
[Documentation] Live end-to-end test for the any-sdk GraphQL
... response.transform path. Queries the public countries
... GraphQL endpoint and asserts the transform flattens
... each continent's nested countries[] array into a
... top-level country_count column. Africa has 58
... countries; the assertion pins that row's projection.
Sleep 2s
${continentsQuery} = Catenate
... select code, name, country_count from trevorblades.geo.continents order by code asc;
${result} = Run Process
... ${STACKQL_EXE}
... \-\-registry
... { "url": "file://${REPOSITORY_ROOT}/test/registry", "localDocRoot": "${REPOSITORY_ROOT}/test/registry", "verifyConfig": { "nopVerify": true } }
... exec
... ${continentsQuery}
... cwd=${REPOSITORY_ROOT}
... stdout=${CURDIR}/tmp/Trevorblades-Continents-GraphQL-Response-Transform.tmp
... stderr=${CURDIR}/tmp/Trevorblades-Continents-GraphQL-Response-Transform-stderr.tmp
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} AF
Should Contain ${result.stdout} Africa
Should Contain ${result.stdout} 58
Loading