Skip to content

Commit a7a6e4d

Browse files
santiagomedsantiagomed
committed
streaming
1 parent 3066947 commit a7a6e4d

4 files changed

Lines changed: 201 additions & 16 deletions

File tree

‎README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,28 @@ Use specific OAuth 2.0 account:
9191
xurl --username johndoe /2/users/me
9292
```
9393

94+
### Streaming Responses
95+
96+
Streaming endpoints (like `/2/tweets/search/stream`) are automatically detected and handled appropriately. The tool will automatically stream the response for these endpoints:
97+
98+
- `/2/tweets/search/stream`
99+
- `/2/tweets/sample/stream`
100+
- `/2/tweets/sample10/stream`
101+
- `/2/tweets/firehose/strea/lang/en`
102+
- `/2/tweets/firehose/stream/lang/ja`
103+
- `/2/tweets/firehose/stream/lang/ko`
104+
- `/2/tweets/firehose/stream/lang/pt`
105+
106+
For example:
107+
```bash
108+
xurl /2/tweets/search/stream
109+
```
110+
111+
You can also force streaming mode for any endpoint using the `--stream` or `-s` flag:
112+
```bash
113+
xurl -s /2/users/me
114+
```
115+
94116
## Token Storage
95117

96118
Tokens are stored securely in `~/.xurl` in your home directory.

‎api/client.go‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"strings"
1111
"time"
1212

13+
"bufio"
1314
"xurl/auth"
1415
"xurl/config"
1516
xurlErrors "xurl/errors"
@@ -93,7 +94,7 @@ func (c *ApiClient) GetAuthHeader(method, url string, authType string, username
9394
return "", err
9495
}
9596
return "Bearer " + token, nil
96-
case "bearer":
97+
case "app":
9798
token := c.auth.BearerToken()
9899
if token == "" {
99100
return "", xurlErrors.NewAuthError("TokenNotFound", errors.New("bearer token not found"))
@@ -251,4 +252,107 @@ func (c *ApiClient) SendRequest(method, endpoint string, headers []string, data
251252
}
252253

253254
return js, nil
255+
}
256+
257+
// StreamRequest sends an HTTP request and streams the response
258+
func (c *ApiClient) StreamRequest(method, endpoint string, headers []string, data string, authType string, username string, verbose bool) *xurlErrors.Error {
259+
req, err := c.BuildRequest(method, endpoint, headers, data, authType, username)
260+
if err != nil {
261+
return xurlErrors.NewHTTPError(err)
262+
}
263+
264+
if verbose {
265+
fmt.Printf("\033[1;34m> %s\033[0m %s\n", req.Method, req.URL)
266+
for key, values := range req.Header {
267+
for _, value := range values {
268+
fmt.Printf("\033[1;36m> %s\033[0m: %s\n", key, value)
269+
}
270+
}
271+
fmt.Println()
272+
}
273+
274+
client := &http.Client{
275+
Timeout: 0,
276+
}
277+
278+
fmt.Printf("\033[1;32mConnecting to streaming endpoint: %s\033[0m\n", endpoint)
279+
280+
resp, err := client.Do(req)
281+
if err != nil {
282+
return xurlErrors.NewHTTPError(err)
283+
}
284+
defer resp.Body.Close()
285+
286+
if verbose {
287+
fmt.Printf("\033[1;31m< %s\033[0m\n", resp.Status)
288+
for key, values := range resp.Header {
289+
for _, value := range values {
290+
fmt.Printf("\033[1;32m< %s\033[0m: %s\n", key, value)
291+
}
292+
}
293+
fmt.Println()
294+
}
295+
296+
if resp.StatusCode >= 400 {
297+
body, err := io.ReadAll(resp.Body)
298+
if err != nil {
299+
return xurlErrors.NewIOError(err)
300+
}
301+
302+
// Check if response is JSON
303+
var js json.RawMessage
304+
if err := json.Unmarshal(body, &js); err != nil {
305+
return xurlErrors.NewJSONError(err)
306+
}
307+
308+
return xurlErrors.NewAPIError(js)
309+
}
310+
311+
contentType := resp.Header.Get("Content-Type")
312+
isJSON := strings.Contains(contentType, "application/json") ||
313+
strings.Contains(contentType, "application/x-ndjson") ||
314+
strings.Contains(contentType, "application/stream+json")
315+
316+
scanner := bufio.NewScanner(resp.Body)
317+
318+
const maxScanTokenSize = 1024 * 1024 // 1MB
319+
buf := make([]byte, maxScanTokenSize)
320+
scanner.Buffer(buf, maxScanTokenSize)
321+
322+
fmt.Println("\033[1;32m--- Streaming response started ---\033[0m")
323+
fmt.Println("\033[1;32m--- Press Ctrl+C to stop ---\033[0m")
324+
325+
for scanner.Scan() {
326+
line := scanner.Text()
327+
328+
if line == "" {
329+
continue
330+
}
331+
332+
if isJSON {
333+
var js json.RawMessage
334+
if err := json.Unmarshal([]byte(line), &js); err != nil {
335+
fmt.Println(line)
336+
} else {
337+
prettyJSON, err := json.MarshalIndent(js, "", " ")
338+
if err != nil {
339+
fmt.Println(line)
340+
} else {
341+
fmt.Println(string(prettyJSON))
342+
}
343+
}
344+
} else {
345+
fmt.Println(line)
346+
}
347+
}
348+
349+
if err := scanner.Err(); err != nil {
350+
if err == bufio.ErrTooLong {
351+
return xurlErrors.NewIOError(fmt.Errorf("line too long: increase buffer size"))
352+
}
353+
return xurlErrors.NewIOError(err)
354+
}
355+
356+
fmt.Println("\033[1;32m--- End of stream ---\033[0m")
357+
return nil
254358
}

‎api/endpoints.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package api
2+
3+
import (
4+
"strings"
5+
)
6+
7+
// StreamingEndpoints is a map of endpoint prefixes that should be streamed
8+
var StreamingEndpoints = map[string]bool{
9+
"/2/tweets/search/stream": true,
10+
"/2/tweets/sample/stream": true,
11+
"/2/tweets/sample10/stream": true,
12+
"/2/tweets/firehose/stream": true,
13+
"/2/tweets/firehose/stream/lang/en": true,
14+
"/2/tweets/firehose/stream/lang/ja": true,
15+
"/2/tweets/firehose/stream/lang/ko": true,
16+
"/2/tweets/firehose/stream/lang/pt": true,
17+
}
18+
19+
// IsStreamingEndpoint checks if an endpoint should be streamed
20+
func IsStreamingEndpoint(endpoint string) bool {
21+
path := endpoint
22+
if strings.HasPrefix(strings.ToLower(endpoint), "http") {
23+
parsedURL := strings.SplitN(endpoint, "/", 4)
24+
if len(parsedURL) >= 4 {
25+
path = "/" + parsedURL[3]
26+
}
27+
}
28+
29+
normalizedEndpoint := strings.TrimSuffix(path, "/")
30+
31+
if StreamingEndpoints[normalizedEndpoint] {
32+
return true
33+
}
34+
35+
for streamingEndpoint := range StreamingEndpoints {
36+
if strings.HasPrefix(normalizedEndpoint, streamingEndpoint) {
37+
return true
38+
}
39+
}
40+
41+
return false
42+
}

‎main.go‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ func main() {
3737
authType, _ := cmd.Flags().GetString("auth")
3838
username, _ := cmd.Flags().GetString("username")
3939
verbose, _ := cmd.Flags().GetBool("verbose")
40+
forceStream, _ := cmd.Flags().GetBool("stream")
4041

4142
// Check if URL is provided
4243
if len(args) == 0 {
@@ -51,23 +52,38 @@ func main() {
5152
// Create API client
5253
client := api.NewApiClient(config, auth)
5354

54-
// Make the request
55-
response, clientErr := client.SendRequest(method, url, headers, data, authType, username, verbose)
56-
if clientErr != nil {
57-
var rawJSON json.RawMessage
58-
json.Unmarshal([]byte(clientErr.Message), &rawJSON)
59-
prettyJSON, _ := json.MarshalIndent(rawJSON, "", " ")
60-
fmt.Println(string(prettyJSON))
61-
os.Exit(1)
62-
}
55+
// Check if the endpoint should be streamed
56+
shouldStream := forceStream || api.IsStreamingEndpoint(url)
6357

64-
// Pretty print the response
65-
prettyJSON, err := json.MarshalIndent(response, "", " ")
66-
if err != nil {
67-
fmt.Println("Error formatting JSON:", err)
68-
os.Exit(1)
58+
if shouldStream {
59+
// Make streaming request
60+
clientErr := client.StreamRequest(method, url, headers, data, authType, username, verbose)
61+
if clientErr != nil {
62+
var rawJSON json.RawMessage
63+
json.Unmarshal([]byte(clientErr.Message), &rawJSON)
64+
prettyJSON, _ := json.MarshalIndent(rawJSON, "", " ")
65+
fmt.Println(string(prettyJSON))
66+
os.Exit(1)
67+
}
68+
} else {
69+
// Make regular request
70+
response, clientErr := client.SendRequest(method, url, headers, data, authType, username, verbose)
71+
if clientErr != nil {
72+
var rawJSON json.RawMessage
73+
json.Unmarshal([]byte(clientErr.Message), &rawJSON)
74+
prettyJSON, _ := json.MarshalIndent(rawJSON, "", " ")
75+
fmt.Println(string(prettyJSON))
76+
os.Exit(1)
77+
}
78+
79+
// Pretty print the response
80+
prettyJSON, err := json.MarshalIndent(response, "", " ")
81+
if err != nil {
82+
fmt.Println("Error formatting JSON:", err)
83+
os.Exit(1)
84+
}
85+
fmt.Println(string(prettyJSON))
6986
}
70-
fmt.Println(string(prettyJSON))
7187
},
7288
}
7389

@@ -78,6 +94,7 @@ func main() {
7894
rootCmd.Flags().String("auth", "", "Authentication type (oauth1 or oauth2)")
7995
rootCmd.Flags().StringP("username", "u", "", "Username for OAuth2 authentication")
8096
rootCmd.Flags().BoolP("verbose", "v", false, "Print verbose information")
97+
rootCmd.Flags().BoolP("stream", "s", false, "Force streaming mode for non-streaming endpoints")
8198

8299
// Create auth command
83100
var authCmd = &cobra.Command{

0 commit comments

Comments
 (0)