Skip to content

Commit f4ecd36

Browse files
mislavmislav
committed
api command: add GraphQL support for --paginate 
1 parent 3f940c9 commit f4ecd36

File tree

4 files changed

+335
-27
lines changed

4 files changed

+335
-27
lines changed

‎pkg/cmd/api/api.go‎

Lines changed: 50 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,11 @@ on the format of the value:
7676
Raw request body may be passed from the outside via a file specified by '--input'.
7777
Pass "-" to read from standard input. In this mode, parameters specified via
7878
'--field' flags are serialized into URL query parameters.
79-
`,
79+
80+
In '--paginate' mode, all pages of results will sequentially be requested until
81+
there are no more pages of results. For GraphQL requests, this requires that the
82+
original query accepts an '$endCursor: String' variable and that it fetches the
83+
'pageInfo{ hasNextPage, endCursor }' set of fields from a collection.`,
8084
Example: heredoc.Doc(`
8185
$ gh api repos/:owner/:repo/releases
8286
@@ -89,6 +93,20 @@ Pass "-" to read from standard input. In this mode, parameters specified via
8993
}
9094
}
9195
'
96+
97+
$ gh api graphql --paginate -f query='
98+
query($endCursor: String) {
99+
viewer {
100+
repositories(first: 100, after: $endCursor) {
101+
nodes { nameWithOwner }
102+
pageInfo {
103+
hasNextPage
104+
endCursor
105+
}
106+
}
107+
}
108+
}
109+
'
92110
`),
93111
Args: cobra.ExactArgs(1),
94112
RunE: func(c *cobra.Command, args []string) error {
@@ -162,15 +180,23 @@ func apiRun(opts *ApiOptions) error {
162180
return err
163181
}
164182

165-
err = processResponse(resp, opts)
183+
endCursor, err := processResponse(resp, opts)
166184
if err != nil {
167185
return err
168186
}
169187

170188
if !opts.Paginate {
171189
break
172190
}
173-
requestPath, hasNextPage = findNextPage(resp)
191+
192+
if opts.RequestPath == "graphql" {
193+
hasNextPage = endCursor != ""
194+
if hasNextPage {
195+
params["endCursor"] = endCursor
196+
}
197+
} else {
198+
requestPath, hasNextPage = findNextPage(resp)
199+
}
174200

175201
if hasNextPage && opts.ShowResponseHeaders {
176202
fmt.Fprint(opts.IO.Out, "\n")
@@ -180,65 +206,63 @@ func apiRun(opts *ApiOptions) error {
180206
return nil
181207
}
182208

183-
var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`)
184-
185-
func findNextPage(resp *http.Response) (string, bool) {
186-
for _, m := range linkRE.FindAllStringSubmatch(resp.Header.Get("Link"), -1) {
187-
if len(m) < 2 {
188-
continue
189-
}
190-
if m[2] == "next" {
191-
return m[1], true
192-
}
193-
}
194-
return "", false
195-
}
196-
197-
func processResponse(resp *http.Response, opts *ApiOptions) error {
209+
func processResponse(resp *http.Response, opts *ApiOptions) (endCursor string, err error) {
198210
if opts.ShowResponseHeaders {
199211
fmt.Fprintln(opts.IO.Out, resp.Proto, resp.Status)
200212
printHeaders(opts.IO.Out, resp.Header, opts.IO.ColorEnabled())
201213
fmt.Fprint(opts.IO.Out, "\r\n")
202214
}
203215

204216
if resp.StatusCode == 204 {
205-
return nil
217+
return
206218
}
207219
var responseBody io.Reader = resp.Body
208220
defer resp.Body.Close()
209221

210222
isJSON, _ := regexp.MatchString(`[/+]json(;|$)`, resp.Header.Get("Content-Type"))
211223

212-
var err error
213224
var serverError string
214225
if isJSON && (opts.RequestPath == "graphql" || resp.StatusCode >= 400) {
215226
responseBody, serverError, err = parseErrorResponse(responseBody, resp.StatusCode)
216227
if err != nil {
217-
return err
228+
return
218229
}
219230
}
220231

232+
var bodyCopy *bytes.Buffer
233+
isGraphQLPaginate := isJSON && resp.StatusCode == 200 && opts.Paginate && opts.RequestPath == "graphql"
234+
if isGraphQLPaginate {
235+
bodyCopy = &bytes.Buffer{}
236+
responseBody = io.TeeReader(responseBody, bodyCopy)
237+
}
238+
221239
if isJSON && opts.IO.ColorEnabled() {
222240
err = jsoncolor.Write(opts.IO.Out, responseBody, " ")
223241
if err != nil {
224-
return err
242+
return
225243
}
226244
} else {
227245
_, err = io.Copy(opts.IO.Out, responseBody)
228246
if err != nil {
229-
return err
247+
return
230248
}
231249
}
232250

233251
if serverError != "" {
234252
fmt.Fprintf(opts.IO.ErrOut, "gh: %s\n", serverError)
235-
return cmdutil.SilentError
253+
err = cmdutil.SilentError
254+
return
236255
} else if resp.StatusCode > 299 {
237256
fmt.Fprintf(opts.IO.ErrOut, "gh: HTTP %d\n", resp.StatusCode)
238-
return cmdutil.SilentError
257+
err = cmdutil.SilentError
258+
return
239259
}
240260

241-
return nil
261+
if isGraphQLPaginate {
262+
endCursor = findEndCursor(bodyCopy)
263+
}
264+
265+
return
242266
}
243267

244268
var placeholderRE = regexp.MustCompile(`\:(owner|repo)\b`)

‎pkg/cmd/api/api_test.go‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package api
22

33
import (
44
"bytes"
5+
"encoding/json"
56
"fmt"
67
"io/ioutil"
78
"net/http"
@@ -13,6 +14,7 @@ import (
1314
"github.com/cli/cli/pkg/iostreams"
1415
"github.com/google/shlex"
1516
"github.com/stretchr/testify/assert"
17+
"github.com/stretchr/testify/require"
1618
)
1719

1820
func Test_NewCmdApi(t *testing.T) {
@@ -293,7 +295,7 @@ func Test_apiRun(t *testing.T) {
293295
}
294296
}
295297

296-
func Test_apiRun_pagination(t *testing.T) {
298+
func Test_apiRun_paginationREST(t *testing.T) {
297299
io, _, stdout, stderr := iostreams.Test()
298300

299301
requestCount := 0
@@ -346,6 +348,83 @@ func Test_apiRun_pagination(t *testing.T) {
346348
assert.Equal(t, "https://api.github.com/repositories/1227/issues?page=3", responses[2].Request.URL.String())
347349
}
348350

351+
func Test_apiRun_paginationGraphQL(t *testing.T) {
352+
io, _, stdout, stderr := iostreams.Test()
353+
354+
requestCount := 0
355+
responses := []*http.Response{
356+
{
357+
StatusCode: 200,
358+
Header: http.Header{"Content-Type": []string{`application/json`}},
359+
Body: ioutil.NopCloser(bytes.NewBufferString(`{
360+
"data": {
361+
"nodes": ["page one"],
362+
"pageInfo": {
363+
"endCursor": "PAGE1_END",
364+
"hasNextPage": true
365+
}
366+
}
367+
}`)),
368+
},
369+
{
370+
StatusCode: 200,
371+
Header: http.Header{"Content-Type": []string{`application/json`}},
372+
Body: ioutil.NopCloser(bytes.NewBufferString(`{
373+
"data": {
374+
"nodes": ["page two"],
375+
"pageInfo": {
376+
"endCursor": "PAGE2_END",
377+
"hasNextPage": false
378+
}
379+
}
380+
}`)),
381+
},
382+
}
383+
384+
options := ApiOptions{
385+
IO: io,
386+
HttpClient: func() (*http.Client, error) {
387+
var tr roundTripper = func(req *http.Request) (*http.Response, error) {
388+
resp := responses[requestCount]
389+
resp.Request = req
390+
requestCount++
391+
return resp, nil
392+
}
393+
return &http.Client{Transport: tr}, nil
394+
},
395+
396+
RequestMethod: "POST",
397+
RequestPath: "graphql",
398+
Paginate: true,
399+
}
400+
401+
err := apiRun(&options)
402+
require.NoError(t, err)
403+
404+
assert.Contains(t, stdout.String(), `"page one"`)
405+
assert.Contains(t, stdout.String(), `"page two"`)
406+
assert.Equal(t, "", stderr.String(), "stderr")
407+
408+
var requestData struct {
409+
Variables map[string]interface{}
410+
}
411+
412+
bb, err := ioutil.ReadAll(responses[0].Request.Body)
413+
require.NoError(t, err)
414+
err = json.Unmarshal(bb, &requestData)
415+
require.NoError(t, err)
416+
_, hasCursor := requestData.Variables["endCursor"].(string)
417+
assert.Equal(t, false, hasCursor)
418+
419+
bb, err = ioutil.ReadAll(responses[1].Request.Body)
420+
require.NoError(t, err)
421+
err = json.Unmarshal(bb, &requestData)
422+
require.NoError(t, err)
423+
endCursor, hasCursor := requestData.Variables["endCursor"].(string)
424+
assert.Equal(t, true, hasCursor)
425+
assert.Equal(t, "PAGE1_END", endCursor)
426+
}
427+
349428
func Test_apiRun_inputFile(t *testing.T) {
350429
tests := []struct {
351430
name string

‎pkg/cmd/api/pagination.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package api
2+
3+
import (
4+
"encoding/json"
5+
"io"
6+
"net/http"
7+
"regexp"
8+
)
9+
10+
var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`)
11+
12+
func findNextPage(resp *http.Response) (string, bool) {
13+
for _, m := range linkRE.FindAllStringSubmatch(resp.Header.Get("Link"), -1) {
14+
if len(m) >= 2 && m[2] == "next" {
15+
return m[1], true
16+
}
17+
}
18+
return "", false
19+
}
20+
21+
func findEndCursor(r io.Reader) string {
22+
dec := json.NewDecoder(r)
23+
24+
var idx int
25+
var stack []json.Delim
26+
var lastKey string
27+
var contextKey string
28+
29+
var endCursor string
30+
var hasNextPage bool
31+
var foundEndCursor bool
32+
var foundNextPage bool
33+
34+
loop:
35+
for {
36+
t, err := dec.Token()
37+
if err == io.EOF {
38+
break
39+
}
40+
if err != nil {
41+
return ""
42+
}
43+
44+
switch tt := t.(type) {
45+
case json.Delim:
46+
switch tt {
47+
case '{', '[':
48+
stack = append(stack, tt)
49+
contextKey = lastKey
50+
idx = 0
51+
case '}', ']':
52+
stack = stack[:len(stack)-1]
53+
contextKey = ""
54+
idx = 0
55+
}
56+
default:
57+
isKey := len(stack) > 0 && stack[len(stack)-1] == '{' && idx%2 == 0
58+
idx++
59+
60+
switch tt := t.(type) {
61+
case string:
62+
if isKey {
63+
lastKey = tt
64+
} else if contextKey == "pageInfo" && lastKey == "endCursor" {
65+
endCursor = tt
66+
foundEndCursor = true
67+
if foundNextPage {
68+
break loop
69+
}
70+
}
71+
case bool:
72+
if contextKey == "pageInfo" && lastKey == "hasNextPage" {
73+
hasNextPage = tt
74+
foundNextPage = true
75+
if foundEndCursor {
76+
break loop
77+
}
78+
}
79+
}
80+
}
81+
}
82+
83+
if hasNextPage {
84+
return endCursor
85+
}
86+
return ""
87+
}

0 commit comments

Comments
 (0)