Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/stackql/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ func init() {
rootCmd.PersistentFlags().BoolVarP(&runtimeCtx.VerboseFlag, dto.VerboseFlagKey, "v", false, "Verbose flag")
rootCmd.PersistentFlags().BoolVar(&runtimeCtx.DryRunFlag, dto.DryRunFlagKey, false, "dryrun flag; preprocessor only will run and output returned")
rootCmd.PersistentFlags().BoolVarP(&runtimeCtx.CSVHeadersDisable, dto.CSVHeadersDisableKey, "H", false, "Disable CSV headers flag")
rootCmd.PersistentFlags().StringVarP(&runtimeCtx.OutputFormat, dto.OutputFormatKey, "o", "table", "Output format, must be (json | table | csv)")
rootCmd.PersistentFlags().StringVarP(&runtimeCtx.OutputFormat, dto.OutputFormatKey, "o", "table", "Output format, must be (json | jsonl | table | csv)")
rootCmd.PersistentFlags().StringVarP(&runtimeCtx.OutfilePath, dto.OutfilePathKey, "f", "stdout", "Output file into which results are written")
rootCmd.PersistentFlags().StringVarP(&runtimeCtx.InfilePath, dto.InfilePathKey, "i", "stdin", "Input file from which queries are read")
rootCmd.PersistentFlags().StringVarP(&runtimeCtx.TemplateCtxFilePath, dto.TemplateCtxFilePathKey, "q", "", "Context file for templating")
Expand Down
78 changes: 77 additions & 1 deletion internal/stackql/output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,13 @@ func GetOutputWriter(
errWriter = os.Stdout
}
ci := pgtype.NewConnInfo()
switch outputCtx.RuntimeContext.OutputFormat {
switch strings.ToLower(outputCtx.RuntimeContext.OutputFormat) {
case constants.JSONStr:
jsonWriter := NewJSONWriter(writer, errWriter)
return jsonWriter, nil
case "jsonl", "ndjson":
jsonlWriter := NewJSONLWriter(writer, errWriter)
return jsonlWriter, nil
case constants.TableStr:
tablewriter := TableWriter{
AbstractTabularWriter{
Expand Down Expand Up @@ -111,13 +114,29 @@ type JSONWriter struct {
errWriter io.Writer
}

type JSONLWriter struct {
writer io.Writer
errWriter io.Writer
}

type writeFlusher interface {
Flush() error
}

func NewJSONWriter(writer io.Writer, errWriter io.Writer) IOutputWriter {
return &JSONWriter{
writer: writer,
errWriter: errWriter,
}
}

func NewJSONLWriter(writer io.Writer, errWriter io.Writer) IOutputWriter {
return &JSONLWriter{
writer: writer,
errWriter: errWriter,
}
}

type AbstractTabularWriter struct {
ci *pgtype.ConnInfo
outputCtx internaldto.OutputContext
Expand Down Expand Up @@ -196,6 +215,63 @@ func (jw *JSONWriter) WriteError(err error, errorPresentation string) error {
return jw.writeRows(rows)
}

func (jw *JSONLWriter) writeRowsFromResult(res sqldata.ISQLResultStream) error {
for {
r, err := res.Read()
logging.GetLogger().Debugln(fmt.Sprintf("result from stream: %v", r))
if err != nil {
if errors.Is(err, io.EOF) {
rowsArr := r.ToArr()
return jw.writeRows(rowsArr)
}
return err
}
rowsArr := r.ToArr()
if err = jw.writeRows(rowsArr); err != nil {
return err
}
}
}

func (jw *JSONLWriter) writeRows(rows []map[string]interface{}) error {
for _, row := range rows {
jsonBytes, jsonErr := json.Marshal(row)
if jsonErr != nil {
return jsonErr
}
bytesWritten, writeErr := jw.writer.Write(append(jsonBytes, '\n'))
if writeErr != nil {
return writeErr
}
if bytesWritten != len(jsonBytes)+1 {
return errors.New("incorrect number of bytes written")
}
if flusher, ok := jw.writer.(writeFlusher); ok {
if flushErr := flusher.Flush(); flushErr != nil {
return flushErr
}
}
}
return nil
}

func (jw *JSONLWriter) Write(res sqldata.ISQLResultStream) error {
return jw.writeRowsFromResult(res)
}

func (jw *JSONLWriter) WriteError(err error, errorPresentation string) error {
if errorPresentation == stderrPressentationStr {
return writeStderrError(jw.errWriter, err)
}
rows := make([]map[string]interface{}, 0, 1)
rows = append(rows,
map[string]interface{}{
errorKey: err.Error(),
},
)
return jw.writeRows(rows)
}

func (tw *AbstractTabularWriter) getHeader(res sqldata.ISQLResult) []string {
var headers []string
for _, col := range res.GetColumns() {
Expand Down
133 changes: 133 additions & 0 deletions internal/stackql/output/output_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package output //nolint:testpackage // do not care

import (
"bytes"
"errors"
"io"
"strings"
"testing"

"github.com/stackql/any-sdk/pkg/dto"
"github.com/stackql/psql-wire/pkg/sqldata"
"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
)

type flushCountingWriter struct {
b bytes.Buffer
flushCount int
}

func (w *flushCountingWriter) Write(p []byte) (int, error) {
return w.b.Write(p)
}

func (w *flushCountingWriter) Flush() error {
w.flushCount++
return nil
}

func (w *flushCountingWriter) String() string {
return w.b.String()
}

type errResultStream struct {
err error
}

func (s *errResultStream) Read() (sqldata.ISQLResult, error) {
return nil, s.err
}

func (s *errResultStream) Write(sqldata.ISQLResult) error {
return nil
}

func (s *errResultStream) Close() error {
return nil
}

func TestJSONLWriter_writeRows(t *testing.T) {
var b bytes.Buffer
w := &JSONLWriter{writer: &b, errWriter: &b}
rows := []map[string]interface{}{
{"k": "v1", "n": 1},
{"k": "v2", "n": 2},
}

if err := w.writeRows(rows); err != nil {
t.Fatalf("writeRows() error = %v", err)
}

out := strings.TrimSpace(b.String())
lines := strings.Split(out, "\n")
if len(lines) != 2 {
t.Fatalf("expected 2 JSONL lines, got %d: %q", len(lines), out)
}
if !strings.Contains(lines[0], `"k":"v1"`) || !strings.Contains(lines[0], `"n":1`) {
t.Fatalf("first line mismatch: %q", lines[0])
}
if !strings.Contains(lines[1], `"k":"v2"`) || !strings.Contains(lines[1], `"n":2`) {
t.Fatalf("second line mismatch: %q", lines[1])
}
}

func TestJSONLWriter_WriteError_Record(t *testing.T) {
var out bytes.Buffer
var errOut bytes.Buffer
w := &JSONLWriter{writer: &out, errWriter: &errOut}

if err := w.WriteError(errors.New("boom"), "record"); err != nil {
t.Fatalf("WriteError() error = %v", err)
}

line := strings.TrimSpace(out.String())
if !strings.Contains(line, `"error":"boom"`) {
t.Fatalf("expected error record in JSONL output, got %q", line)
}
if errOut.Len() != 0 {
t.Fatalf("expected no stderr output, got %q", errOut.String())
}
}

func TestGetOutputWriter_JSONL(t *testing.T) {
ctx := internaldto.OutputContext{RuntimeContext: dto.RuntimeCtx{OutputFormat: "jsonl"}}
w, err := GetOutputWriter(&bytes.Buffer{}, &bytes.Buffer{}, ctx)
if err != nil {
t.Fatalf("GetOutputWriter() error = %v", err)
}
if _, ok := w.(*JSONLWriter); !ok {
t.Fatalf("expected *JSONLWriter, got %T", w)
}
}

func TestJSONLWriter_writeRows_EagerFlushPerRow(t *testing.T) {
target := &flushCountingWriter{}
w := &JSONLWriter{writer: target, errWriter: io.Discard}
rows := []map[string]interface{}{
{"k": "v1", "n": 1},
{"k": "v2", "n": 2},
}

if err := w.writeRows(rows); err != nil {
t.Fatalf("writeRows() error = %v", err)
}

if target.flushCount != len(rows) {
t.Fatalf("expected flush count %d, got %d", len(rows), target.flushCount)
}

out := strings.TrimSpace(target.String())
if got := len(strings.Split(out, "\n")); got != len(rows) {
t.Fatalf("expected %d lines, got %d: %q", len(rows), got, out)
}
}

func TestJSONLWriter_Write_PropagatesStreamError(t *testing.T) {
w := &JSONLWriter{writer: &bytes.Buffer{}, errWriter: io.Discard}
wantErr := errors.New("stream broke")

err := w.Write(&errResultStream{err: wantErr})
if !errors.Is(err, wantErr) {
t.Fatalf("expected %v, got %v", wantErr, err)
}
}
85 changes: 85 additions & 0 deletions test/robot/functional/stackql_from_cmd_line.robot
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ Get Providers
Should StackQL Novel Exec Contain ${SHOW_PROVIDERS_STR} v0.3.1
Should StackQL Exec Contain JSON output ${SHOW_PROVIDERS_STR} okta

Get Providers JSONL
Should StackQL Exec Contain JSONL output ${SHOW_PROVIDERS_STR} "name":"okta"

AWS VPN Gateways Empty JSONL Exact
[Documentation] Deterministic mock-backed query with ORDER BY; expect exact empty JSONL output.
Should StackQL Exec Equal JSONL output ${SELECT_AWS_EC2_VPN_GATEWAYS_NULL} ${EMPTY}

Get Providers No Config
Should StackQL No Cfg Exec Contain ${SHOW_PROVIDERS_STR} name

Expand Down Expand Up @@ -954,6 +961,84 @@ Should StackQL Exec Contain JSON output
... \-o\=json
... &{kwargs}

Should StackQL Exec Contain JSONL output
[Arguments] ${_EXEC_CMD_STR} ${_EXEC_CMD_EXPECTED_OUTPUT} @{varargs} &{kwargs}
Should StackQL Exec Inline Contain
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_NO_VERIFY_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${_EXEC_CMD_STR}
... ${_EXEC_CMD_EXPECTED_OUTPUT}
... \-o\=jsonl
... &{kwargs}
Should StackQL Exec Inline Contain
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_CANONICAL_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${_EXEC_CMD_STR}
... ${_EXEC_CMD_EXPECTED_OUTPUT}
... \-o\=jsonl
... &{kwargs}
Should StackQL Exec Inline Contain
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_DEPRECATED_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${_EXEC_CMD_STR}
... ${_EXEC_CMD_EXPECTED_OUTPUT}
... \-o\=jsonl
... &{kwargs}

Should StackQL Exec Equal JSONL output
[Arguments] ${_EXEC_CMD_STR} ${_EXEC_CMD_EXPECTED_OUTPUT} @{varargs} &{kwargs}
Should StackQL Exec Inline Equal
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_NO_VERIFY_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${_EXEC_CMD_STR}
... ${_EXEC_CMD_EXPECTED_OUTPUT}
... \-o\=jsonl
... &{kwargs}
Should StackQL Exec Inline Equal
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_CANONICAL_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${_EXEC_CMD_STR}
... ${_EXEC_CMD_EXPECTED_OUTPUT}
... \-o\=jsonl
... &{kwargs}
Should StackQL Exec Inline Equal
... ${STACKQL_EXE}
... ${OKTA_SECRET_STR}
... ${GITHUB_SECRET_STR}
... ${K8S_SECRET_STR}
... ${REGISTRY_DEPRECATED_CFG_STR}
... ${AUTH_CFG_STR}
... ${SQL_BACKEND_CFG_STR_CANONICAL}
... ${_EXEC_CMD_STR}
... ${_EXEC_CMD_EXPECTED_OUTPUT}
... \-o\=jsonl
... &{kwargs}

Should StackQL Novel Exec Contain
[Arguments] ${_EXEC_CMD_STR} ${_EXEC_CMD_EXPECTED_OUTPUT} @{varargs} &{kwargs}
Should StackQL Exec Inline Contain
Expand Down
Loading