Skip to content

Commit af095d5

Browse files
AchoArnoldAchoArnold
committed
Fix the dispatch to handle situations when google cloud task has an outage
1 parent 939ce20 commit af095d5

4 files changed

Lines changed: 31 additions & 19 deletions

File tree

‎api/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ require (
4646
go.opentelemetry.io/otel/sdk/metric v0.39.0
4747
go.opentelemetry.io/otel/trace v1.16.0
4848
google.golang.org/api v0.134.0
49-
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e
5049
google.golang.org/protobuf v1.31.0
5150
gorm.io/datatypes v1.2.0
5251
gorm.io/driver/postgres v1.5.2
@@ -151,6 +150,7 @@ require (
151150
golang.org/x/tools v0.11.0 // indirect
152151
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
153152
google.golang.org/appengine v1.6.7 // indirect
153+
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e // indirect
154154
google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect
155155
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e // indirect
156156
google.golang.org/grpc v1.57.0 // indirect

‎api/pkg/services/discord_service.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ func (service *DiscordService) sendMessage(ctx context.Context, event cloudevent
247247
message, response, err := service.client.Channel.CreateMessage(ctx, discord.IncomingChannelID, request)
248248
if err != nil {
249249
msg := fmt.Sprintf("cannot send [%s] event to discord channel [%s] for user [%s]", event.Type(), discord.IncomingChannelID, discord.UserID)
250-
ctxLogger.Error(service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg)))
250+
ctxLogger.Warn(service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg)))
251251

252252
eventPayload := &events.DiscordSendFailedPayload{
253253
DiscordID: discord.ID,

‎api/pkg/services/event_dispatcher_service.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package services
33
import (
44
"context"
55
"encoding/json"
6+
"errors"
67
"fmt"
78
"net/http"
89
"sync"
@@ -61,7 +62,7 @@ func (dispatcher *EventDispatcher) DispatchSync(ctx context.Context, event cloud
6162

6263
// DispatchWithTimeout dispatches an event with a timeout
6364
func (dispatcher *EventDispatcher) DispatchWithTimeout(ctx context.Context, event cloudevents.Event, timeout time.Duration) (queueID string, err error) {
64-
ctx, span := dispatcher.tracer.Start(ctx)
65+
ctx, span, ctxLogger := dispatcher.tracer.StartWithLogger(ctx, dispatcher.logger)
6566
defer span.End()
6667

6768
if err := event.Validate(); err != nil {
@@ -75,7 +76,17 @@ func (dispatcher *EventDispatcher) DispatchWithTimeout(ctx context.Context, even
7576
return queueID, dispatcher.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
7677
}
7778

78-
if queueID, err = dispatcher.queue.Enqueue(ctx, task, timeout); err != nil {
79+
queueID, err = dispatcher.queue.Enqueue(ctx, task, timeout)
80+
if errors.Is(err, context.DeadlineExceeded) {
81+
msg := fmt.Sprintf("cannot enqueue event with ID [%s] and type [%s]. publishing locally", event.ID(), event.Type())
82+
ctxLogger.Warn(stacktrace.Propagate(err, msg))
83+
queueID, err = fmt.Sprintf("local-%s", event.ID()), nil
84+
time.AfterFunc(timeout, func() {
85+
dispatcher.Publish(ctx, event)
86+
})
87+
}
88+
89+
if err != nil {
7990
msg := fmt.Sprintf("cannot enqueue event with ID [%s] and type [%s]", event.ID(), event.Type())
8091
return queueID, dispatcher.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
8192
}

‎api/pkg/services/google_cloud_push_queue_service.go‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ import (
77
"time"
88

99
cloudtasks "cloud.google.com/go/cloudtasks/apiv2"
10+
"cloud.google.com/go/cloudtasks/apiv2/cloudtaskspb"
1011
"github.com/NdoleStudio/httpsms/pkg/telemetry"
1112
"github.com/palantir/stacktrace"
12-
taskspb "google.golang.org/genproto/googleapis/cloud/tasks/v2"
1313
"google.golang.org/protobuf/types/known/timestamppb"
1414
)
1515

@@ -48,13 +48,11 @@ func (queue *googlePushQueue) Enqueue(ctx context.Context, task *PushQueueTask,
4848
}
4949

5050
// Build the Task payload.
51-
// https://godoc.org/google.golang.org/genproto/googleapis/cloud/tasks/v2#CreateTaskRequest
52-
req := &taskspb.CreateTaskRequest{
51+
req := &cloudtaskspb.CreateTaskRequest{
5352
Parent: queue.queueConfig.Name,
54-
Task: &taskspb.Task{
55-
// https://godoc.org/google.golang.org/genproto/googleapis/cloud/tasks/v2#HttpRequest
56-
MessageType: &taskspb.Task_HttpRequest{
57-
HttpRequest: &taskspb.HttpRequest{
53+
Task: &cloudtaskspb.Task{
54+
MessageType: &cloudtaskspb.Task_HttpRequest{
55+
HttpRequest: &cloudtaskspb.HttpRequest{
5856
Headers: headers,
5957
HttpMethod: queue.httpMethodToProtoHTTPMethod(task.Method),
6058
Url: task.URL,
@@ -67,9 +65,12 @@ func (queue *googlePushQueue) Enqueue(ctx context.Context, task *PushQueueTask,
6765
// Add a payload message if one is present.
6866
req.Task.GetHttpRequest().Body = task.Body
6967

70-
queueTask, err := queue.client.CreateTask(ctx, req)
68+
requestCtx, cancel := context.WithTimeout(ctx, time.Second)
69+
defer cancel()
70+
71+
queueTask, err := queue.client.CreateTask(requestCtx, req)
7172
if err != nil {
72-
msg := fmt.Sprintf("cannot schedule task %s to URL: %s", string(task.Body), task.URL)
73+
msg := fmt.Sprintf("cannot schedule task [%s] to URL [%s]", string(task.Body), task.URL)
7374
return queueID, queue.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
7475
}
7576

@@ -83,15 +84,15 @@ func (queue *googlePushQueue) Enqueue(ctx context.Context, task *PushQueueTask,
8384
return queueTask.Name, nil
8485
}
8586

86-
func (queue *googlePushQueue) httpMethodToProtoHTTPMethod(httpMethod string) taskspb.HttpMethod {
87-
method, ok := map[string]taskspb.HttpMethod{
88-
http.MethodGet: taskspb.HttpMethod_GET,
89-
http.MethodPost: taskspb.HttpMethod_POST,
90-
http.MethodPut: taskspb.HttpMethod_PUT,
87+
func (queue *googlePushQueue) httpMethodToProtoHTTPMethod(httpMethod string) cloudtaskspb.HttpMethod {
88+
method, ok := map[string]cloudtaskspb.HttpMethod{
89+
http.MethodGet: cloudtaskspb.HttpMethod_GET,
90+
http.MethodPost: cloudtaskspb.HttpMethod_POST,
91+
http.MethodPut: cloudtaskspb.HttpMethod_PUT,
9192
}[httpMethod]
9293

9394
if !ok {
94-
return taskspb.HttpMethod_POST
95+
return cloudtaskspb.HttpMethod_POST
9596
}
9697

9798
return method

0 commit comments

Comments
 (0)