Skip to content

Commit 866df9a

Browse files
AchoArnoldAchoArnold
committed
Add metric when an event is processed
1 parent 7ed6db8 commit 866df9a

9 files changed

Lines changed: 176 additions & 28 deletions

‎api/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ require (
4141
github.com/uptrace/uptrace-go v1.16.0
4242
github.com/xuri/excelize/v2 v2.8.0
4343
go.opentelemetry.io/otel v1.16.0
44+
go.opentelemetry.io/otel/metric v1.16.0
4445
go.opentelemetry.io/otel/sdk v1.16.0
4546
go.opentelemetry.io/otel/sdk/metric v0.39.0
4647
go.opentelemetry.io/otel/trace v1.16.0
@@ -136,7 +137,6 @@ require (
136137
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
137138
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect
138139
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect
139-
go.opentelemetry.io/otel/metric v1.16.0 // indirect
140140
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
141141
go.uber.org/atomic v1.11.0 // indirect
142142
go.uber.org/multierr v1.11.0 // indirect

‎api/pkg/di/container.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99
"strconv"
1010
"time"
1111

12+
otelMetric "go.opentelemetry.io/otel/metric"
13+
1214
"github.com/dgraph-io/ristretto"
1315

1416
"github.com/gofiber/contrib/otelfiber"
@@ -574,6 +576,7 @@ func (container *Container) EventDispatcher() (dispatcher *services.EventDispatc
574576
dispatcher = services.NewEventDispatcher(
575577
container.Logger(),
576578
container.Tracer(),
579+
container.Float64Histogram("event.publisher.duration", "ms", "measures the duration of processing CloudEvents"),
577580
container.EventsQueue(),
578581
container.EventsQueueConfiguration(),
579582
)
@@ -582,6 +585,20 @@ func (container *Container) EventDispatcher() (dispatcher *services.EventDispatc
582585
return dispatcher
583586
}
584587

588+
// Float64Histogram creates a new instance of metric.Float64Histogram
589+
func (container *Container) Float64Histogram(name, unit, description string) otelMetric.Float64Histogram {
590+
container.logger.Debug("creating GORM repositories.MessageRepository")
591+
meter := otel.GetMeterProvider().Meter(
592+
container.projectID,
593+
otelMetric.WithInstrumentationVersion(otel.Version()),
594+
)
595+
histogram, err := meter.Float64Histogram(name, otelMetric.WithUnit(unit), otelMetric.WithDescription(description))
596+
if err != nil {
597+
container.logger.Fatal(stacktrace.Propagate(err, "cannot create float64 histogram"))
598+
}
599+
return histogram
600+
}
601+
585602
// MessageRepository creates a new instance of repositories.MessageRepository
586603
func (container *Container) MessageRepository() (repository repositories.MessageRepository) {
587604
container.logger.Debug("creating GORM repositories.MessageRepository")

‎api/pkg/emails/hermes_notification_email_factory.go‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"fmt"
55
"time"
66

7+
"github.com/NdoleStudio/httpsms/pkg/events"
8+
79
"github.com/google/uuid"
810

911
"github.com/NdoleStudio/httpsms/pkg/entities"
@@ -25,19 +27,19 @@ func NewHermesNotificationEmailFactory(config *HermesGeneratorConfig) Notificati
2527
}
2628
}
2729

28-
func (factory *hermesNotificationEmailFactory) DiscordMessageFailed(user *entities.User, eventName, owner, errorMessage, channelID string, httpResponseStatusCode *int) (*Email, error) {
30+
func (factory *hermesNotificationEmailFactory) DiscordSendFailed(user *entities.User, payload *events.DiscordSendFailedPayload) (*Email, error) {
2931
email := hermes.Email{
3032
Body: hermes.Body{
3133
Title: "Hello",
3234
Intros: []string{
3335
fmt.Sprintf("We ran into an error while fowarding an incoming SMS to your discord server at %s", user.UserTimeString(time.Now())),
3436
},
3537
Dictionary: []hermes.Entry{
36-
{"Discord Channel ID", channelID},
37-
{"Event Name", eventName},
38-
{"Phone Number", factory.formatPhoneNumber(owner)},
39-
{"HTTP Response Code", factory.formatHTTPResponseCode(httpResponseStatusCode)},
40-
{"Error Message / HTTP Response", errorMessage},
38+
{"Discord Channel ID", payload.DiscordChannelID},
39+
{"Event Name", payload.EventType},
40+
{"Phone Number", factory.formatPhoneNumber(payload.Owner)},
41+
{"HTTP Response Code", factory.formatHTTPResponseCode(payload.HTTPResponseStatusCode)},
42+
{"Error Message / HTTP Response", payload.ErrorMessage},
4143
},
4244
Actions: []hermes.Action{
4345
{
@@ -75,20 +77,20 @@ func (factory *hermesNotificationEmailFactory) DiscordMessageFailed(user *entiti
7577
}, nil
7678
}
7779

78-
func (factory *hermesNotificationEmailFactory) WebhookSendFailed(user *entities.User, eventName, eventID, owner, errorMessage, url string, httpResponseStatusCode *int) (*Email, error) {
80+
func (factory *hermesNotificationEmailFactory) WebhookSendFailed(user *entities.User, payload *events.WebhookSendFailedPayload) (*Email, error) {
7981
email := hermes.Email{
8082
Body: hermes.Body{
8183
Title: "Hello",
8284
Intros: []string{
8385
fmt.Sprintf("We ran into an error while fowarding a webhook event from httpSMS to your webserver at %s", user.UserTimeString(time.Now())),
8486
},
8587
Dictionary: []hermes.Entry{
86-
{"Server URL", url},
87-
{"Event Name", eventName},
88-
{"Event ID", eventID},
89-
{"Phone Number", factory.formatPhoneNumber(owner)},
90-
{"HTTP Response Code", factory.formatHTTPResponseCode(httpResponseStatusCode)},
91-
{"Error Message / HTTP Response", errorMessage},
88+
{"Server URL", payload.WebhookURL},
89+
{"Event Name", payload.EventType},
90+
{"Event ID", payload.EventID},
91+
{"Phone Number", factory.formatPhoneNumber(payload.Owner)},
92+
{"HTTP Response Code", factory.formatHTTPResponseCode(payload.HTTPResponseStatusCode)},
93+
{"Error Message / HTTP Response", payload.ErrorMessage},
9294
},
9395
Actions: []hermes.Action{
9496
{

‎api/pkg/emails/notification_email_factory.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package emails
22

33
import (
44
"github.com/NdoleStudio/httpsms/pkg/entities"
5+
"github.com/NdoleStudio/httpsms/pkg/events"
56
"github.com/google/uuid"
67
)
78

@@ -14,8 +15,8 @@ type NotificationEmailFactory interface {
1415
MessageFailed(user *entities.User, messageID uuid.UUID, owner, contact, content, reason string) (*Email, error)
1516

1617
// DiscordMessageFailed sends an email when the user's discord message is failed
17-
DiscordMessageFailed(user *entities.User, eventName, owner, errorMessage, channelID string, responseHTTPStatusCode *int) (*Email, error)
18+
DiscordSendFailed(user *entities.User, payload *events.DiscordSendFailedPayload) (*Email, error)
1819

1920
// WebhookSendFailed sends an email when the user's webhook message is failed
20-
WebhookSendFailed(user *entities.User, eventName, eventID, owner, errorMessage, url string, responseHTTPStatusCode *int) (*Email, error)
21+
WebhookSendFailed(user *entities.User, payload *events.WebhookSendFailedPayload) (*Email, error)
2122
}

‎api/pkg/events/discord_message_failed_event.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ import (
55
"github.com/google/uuid"
66
)
77

8-
// EventTypeDiscordMessageFailed is emitted when we can't send a discord message
9-
const EventTypeDiscordMessageFailed = "discord.message.failed"
8+
// EventTypeDiscordSendFailed is emitted when we can't send a discord message
9+
const EventTypeDiscordSendFailed = "discord.send.failed"
1010

11-
// DiscordMessageFailedPayload is the payload of the EventTypeDiscordMessageFailed event
12-
type DiscordMessageFailedPayload struct {
11+
// DiscordSendFailedPayload is the payload of the EventTypeDiscordSendFailed event
12+
type DiscordSendFailedPayload struct {
1313
DiscordID uuid.UUID `json:"discord_id"`
1414
UserID entities.UserID `json:"user_id"`
1515
MessageID uuid.UUID `json:"message_id"`

‎api/pkg/listeners/email_notification_listener.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ func NewEmailNotificationListener(
3333
return l, map[string]events.EventListener{
3434
events.EventTypeMessageSendExpired: l.OnMessageSendExpired,
3535
events.EventTypeMessageSendFailed: l.OnMessageSendFailed,
36+
events.EventTypeWebhookSendFailed: l.OnWebhookSendFailed,
37+
events.EventTypeDiscordSendFailed: l.OnDiscordSendFailed,
3638
}
3739
}
3840

@@ -73,3 +75,41 @@ func (listener *EmailNotificationListener) OnMessageSendFailed(ctx context.Conte
7375

7476
return nil
7577
}
78+
79+
// OnWebhookSendFailed handles the events.EventTypeWebhookSendFailed event
80+
func (listener *EmailNotificationListener) OnWebhookSendFailed(ctx context.Context, event cloudevents.Event) error {
81+
ctx, span := listener.tracer.Start(ctx)
82+
defer span.End()
83+
84+
payload := new(events.WebhookSendFailedPayload)
85+
if err := event.DataAs(&payload); err != nil {
86+
msg := fmt.Sprintf("cannot decode [%s] into [%T]", event.Data(), payload)
87+
return listener.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
88+
}
89+
90+
if err := listener.service.NotifyWebhookSendFailed(ctx, payload); err != nil {
91+
msg := fmt.Sprintf("cannot process [%s] event with ID [%s]", event.Type(), event.ID())
92+
return listener.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
93+
}
94+
95+
return nil
96+
}
97+
98+
// OnDiscordSendFailed handles the events.EventTypeDiscordSendFailed event
99+
func (listener *EmailNotificationListener) OnDiscordSendFailed(ctx context.Context, event cloudevents.Event) error {
100+
ctx, span := listener.tracer.Start(ctx)
101+
defer span.End()
102+
103+
payload := new(events.DiscordSendFailedPayload)
104+
if err := event.DataAs(&payload); err != nil {
105+
msg := fmt.Sprintf("cannot decode [%s] into [%T]", event.Data(), payload)
106+
return listener.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
107+
}
108+
109+
if err := listener.service.NotifyDiscordSendFailed(ctx, payload); err != nil {
110+
msg := fmt.Sprintf("cannot process [%s] event with ID [%s]", event.Type(), event.ID())
111+
return listener.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
112+
}
113+
114+
return nil
115+
}

‎api/pkg/services/discord_service.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ func (service *DiscordService) sendMessage(ctx context.Context, event cloudevent
249249
msg := fmt.Sprintf("cannot send [%s] event to discord channel [%s] for user [%s]", event.Type(), discord.IncomingChannelID, discord.UserID)
250250
ctxLogger.Error(service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg)))
251251

252-
eventPayload := &events.DiscordMessageFailedPayload{
252+
eventPayload := &events.DiscordSendFailedPayload{
253253
DiscordID: discord.ID,
254254
UserID: discord.UserID,
255255
MessageID: payload.MessageID,
@@ -301,13 +301,13 @@ func (service *DiscordService) createDiscordMessage(ctxLogger telemetry.Logger,
301301
}
302302
}
303303

304-
func (service *DiscordService) handleDiscordMessageFailed(ctx context.Context, source string, payload *events.DiscordMessageFailedPayload) {
304+
func (service *DiscordService) handleDiscordMessageFailed(ctx context.Context, source string, payload *events.DiscordSendFailedPayload) {
305305
ctx, span, ctxLogger := service.tracer.StartWithLogger(ctx, service.logger)
306306
defer span.End()
307307

308-
event, err := service.createEvent(events.EventTypeDiscordMessageFailed, source, payload)
308+
event, err := service.createEvent(events.EventTypeDiscordSendFailed, source, payload)
309309
if err != nil {
310-
msg := fmt.Sprintf("cannot create event [%s] for user with id [%s]", events.EventTypeDiscordMessageFailed, payload.UserID)
310+
msg := fmt.Sprintf("cannot create event [%s] for user with id [%s]", events.EventTypeDiscordSendFailed, payload.UserID)
311311
ctxLogger.Error(service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg)))
312312
return
313313
}

‎api/pkg/services/email_notification_service.go‎

Lines changed: 75 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ type EmailNotificationService struct {
2525
cache cache.Cache
2626
}
2727

28+
const (
29+
fifteenMinuteTimeout = 15 * time.Minute
30+
oneHourTimeout = 1 * time.Hour
31+
)
32+
2833
// NewEmailNotificationService creates a new EmailNotificationService
2934
func NewEmailNotificationService(
3035
logger telemetry.Logger,
@@ -78,7 +83,7 @@ func (service *EmailNotificationService) NotifyMessageExpired(ctx context.Contex
7883

7984
ctxLogger.Info(fmt.Sprintf("[%s] email sent to [%s] for message with ID [%s]", events.EventTypeMessageSendExpired, user.ID, payload.MessageID))
8085

81-
service.addToCache(ctx, events.EventTypeMessageSendExpired, payload.Owner)
86+
service.addToCache(ctx, fifteenMinuteTimeout, events.EventTypeMessageSendExpired, payload.Owner)
8287
return nil
8388
}
8489

@@ -111,7 +116,73 @@ func (service *EmailNotificationService) NotifyMessageFailed(ctx context.Context
111116

112117
ctxLogger.Info(fmt.Sprintf("[%s] email sent to [%s] for message with ID [%s]", events.EventTypeMessageSendFailed, user.ID, payload.ID))
113118

114-
service.addToCache(ctx, events.EventTypeMessageSendFailed, payload.Owner)
119+
service.addToCache(ctx, fifteenMinuteTimeout, events.EventTypeMessageSendFailed, payload.Owner)
120+
return nil
121+
}
122+
123+
// NotifyWebhookSendFailed sends an email to the user about a failed webhook
124+
func (service *EmailNotificationService) NotifyWebhookSendFailed(ctx context.Context, payload *events.WebhookSendFailedPayload) error {
125+
ctx, span, ctxLogger := service.tracer.StartWithLogger(ctx, service.logger)
126+
defer span.End()
127+
128+
if !service.canSendEmail(ctx, payload.EventType, payload.Owner) {
129+
ctxLogger.Info(fmt.Sprintf("[%s] email already sent to user [%s] with owner [%s]", events.EventTypeWebhookSendFailed, payload.UserID, payload.Owner))
130+
return nil
131+
}
132+
133+
user, err := service.userRepository.Load(ctx, payload.UserID)
134+
if err != nil {
135+
msg := fmt.Sprintf("cannot load user with ID [%s] for [%s] event with ID [%s]", payload.UserID, events.EventTypeWebhookSendFailed, payload.EventID)
136+
return service.tracer.WrapErrorSpan(span, stacktrace.PropagateWithCode(err, stacktrace.GetCode(err), msg))
137+
}
138+
139+
email, err := service.factory.WebhookSendFailed(user, payload)
140+
if err != nil {
141+
msg := fmt.Sprintf("cannot create [%s] email for user with ID [%s] for [%s] event with ID [%s]", events.EventTypeWebhookSendFailed, payload.UserID, payload.EventType, payload.EventID)
142+
return service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
143+
}
144+
145+
if err = service.mailer.Send(ctx, email); err != nil {
146+
msg := fmt.Sprintf("cannot send [%s] email for user with ID [%s] for [%s] event with ID [%s]", events.EventTypeWebhookSendFailed, payload.UserID, payload.EventType, payload.EventID)
147+
return service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
148+
}
149+
150+
ctxLogger.Info(fmt.Sprintf("[%s] email sent to [%s] for [%s] event with ID [%s]", events.EventTypeWebhookSendFailed, user.ID, payload.EventType, payload.EventID))
151+
152+
service.addToCache(ctx, oneHourTimeout, events.EventTypeWebhookSendFailed, payload.Owner)
153+
return nil
154+
}
155+
156+
// NotifyDiscordSendFailed sends an email to the user about a failed discord webhook event
157+
func (service *EmailNotificationService) NotifyDiscordSendFailed(ctx context.Context, payload *events.DiscordSendFailedPayload) error {
158+
ctx, span, ctxLogger := service.tracer.StartWithLogger(ctx, service.logger)
159+
defer span.End()
160+
161+
if !service.canSendEmail(ctx, payload.EventType, payload.Owner) {
162+
ctxLogger.Info(fmt.Sprintf("[%s] email already sent to user [%s] with owner [%s]", events.EventTypeWebhookSendFailed, payload.UserID, payload.Owner))
163+
return nil
164+
}
165+
166+
user, err := service.userRepository.Load(ctx, payload.UserID)
167+
if err != nil {
168+
msg := fmt.Sprintf("cannot load user with ID [%s] for [%s] event for message with ID [%s]", payload.UserID, payload.EventType, payload.MessageID)
169+
return service.tracer.WrapErrorSpan(span, stacktrace.PropagateWithCode(err, stacktrace.GetCode(err), msg))
170+
}
171+
172+
email, err := service.factory.DiscordSendFailed(user, payload)
173+
if err != nil {
174+
msg := fmt.Sprintf("cannot create email for user with ID [%s] for [%s] event and message with ID [%s]", payload.UserID, payload.EventType, payload.MessageID)
175+
return service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
176+
}
177+
178+
if err = service.mailer.Send(ctx, email); err != nil {
179+
msg := fmt.Sprintf("cannot send email for user with ID [%s] for [%s] message with ID [%s]", payload.UserID, events.EventTypeMessageSendFailed, payload.MessageID)
180+
return service.tracer.WrapErrorSpan(span, stacktrace.Propagate(err, msg))
181+
}
182+
183+
ctxLogger.Info(fmt.Sprintf("[%s] email sent to [%s] for [%s] event with ID [%s]", payload.EventType, user.ID, payload.EventType, payload.MessageID))
184+
185+
service.addToCache(ctx, oneHourTimeout, events.EventTypeDiscordSendFailed, payload.Owner)
115186
return nil
116187
}
117188

@@ -124,12 +195,12 @@ func (service *EmailNotificationService) canSendEmail(ctx context.Context, event
124195
return err != nil
125196
}
126197

127-
func (service *EmailNotificationService) addToCache(ctx context.Context, event string, owner string) {
198+
func (service *EmailNotificationService) addToCache(ctx context.Context, timeout time.Duration, event string, owner string) {
128199
ctx, span, ctxLogger := service.tracer.StartWithLogger(ctx, service.logger)
129200
defer span.End()
130201

131202
cacheKey := service.getCacheKey(event, owner)
132-
if err := service.cache.Set(ctx, cacheKey, "", time.Minute*15); err != nil {
203+
if err := service.cache.Set(ctx, cacheKey, "", timeout); err != nil {
133204
ctxLogger.Error(stacktrace.Propagate(err, fmt.Sprintf("cannot set item in redis with key [%s] for owner [%s]", cacheKey, owner)))
134205
}
135206
}

‎api/pkg/services/event_dispatcher_service.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ import (
88
"sync"
99
"time"
1010

11+
"go.opentelemetry.io/otel/metric"
12+
semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
13+
1114
"github.com/NdoleStudio/httpsms/pkg/events"
1215
"github.com/NdoleStudio/httpsms/pkg/telemetry"
1316
cloudevents "github.com/cloudevents/sdk-go/v2"
@@ -19,6 +22,7 @@ type EventDispatcher struct {
1922
logger telemetry.Logger
2023
tracer telemetry.Tracer
2124
listeners map[string][]events.EventListener
25+
meter metric.Float64Histogram
2226
queue PushQueue
2327
queueConfig PushQueueConfig
2428
}
@@ -27,12 +31,14 @@ type EventDispatcher struct {
2731
func NewEventDispatcher(
2832
logger telemetry.Logger,
2933
tracer telemetry.Tracer,
34+
meter metric.Float64Histogram,
3035
queue PushQueue,
3136
queueConfig PushQueueConfig,
3237
) (dispatcher *EventDispatcher) {
3338
return &EventDispatcher{
3439
logger: logger,
3540
tracer: tracer,
41+
meter: meter,
3642
listeners: make(map[string][]events.EventListener),
3743
queue: queue,
3844
queueConfig: queueConfig,
@@ -99,6 +105,8 @@ func (dispatcher *EventDispatcher) Publish(ctx context.Context, event cloudevent
99105
ctx, span := dispatcher.tracer.Start(ctx)
100106
defer span.End()
101107

108+
start := time.Now()
109+
102110
ctxLogger := dispatcher.tracer.CtxLogger(dispatcher.logger, span)
103111

104112
subscribers, ok := dispatcher.listeners[event.Type()]
@@ -120,6 +128,15 @@ func (dispatcher *EventDispatcher) Publish(ctx context.Context, event cloudevent
120128
}
121129

122130
wg.Wait()
131+
132+
dispatcher.meter.Record(
133+
ctx,
134+
float64(time.Since(start).Microseconds())/1000,
135+
metric.WithAttributes(
136+
semconv.CloudeventsEventType(event.Type()),
137+
semconv.CloudeventsEventSource(event.Source()),
138+
),
139+
)
123140
}
124141

125142
func (dispatcher *EventDispatcher) createCloudTask(event cloudevents.Event) (*PushQueueTask, error) {

0 commit comments

Comments
 (0)