Skip to content

Commit f2a0ab8

Browse files
whiteboxsolutionswhiteboxsolutions
committed
Added cache for csv uploads to avoid 1000s of duplicate http requests
1 parent 40f718c commit f2a0ab8

3 files changed

Lines changed: 42 additions & 10 deletions

File tree

‎api/pkg/validators/bulk_message_handler_validator.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/NdoleStudio/httpsms/pkg/requests"
1818
"github.com/NdoleStudio/httpsms/pkg/services"
1919
"github.com/NdoleStudio/httpsms/pkg/telemetry"
20+
"github.com/NdoleStudio/httpsms/pkg/cache"
2021
"github.com/dustin/go-humanize"
2122
"github.com/jszwec/csvutil"
2223
"github.com/nyaruka/phonenumbers"
@@ -30,6 +31,7 @@ type BulkMessageHandlerValidator struct {
3031
userService *services.UserService
3132
logger telemetry.Logger
3233
tracer telemetry.Tracer
34+
cache cache.Cache
3335
}
3436

3537
// NewBulkMessageHandlerValidator creates a new handlers.BulkMessageHandlerValidator validator
@@ -38,12 +40,14 @@ func NewBulkMessageHandlerValidator(
3840
tracer telemetry.Tracer,
3941
phoneService *services.PhoneService,
4042
userService *services.UserService,
43+
appCache cache.Cache,
4144
) (v *BulkMessageHandlerValidator) {
4245
return &BulkMessageHandlerValidator{
4346
logger: logger.WithService(fmt.Sprintf("%T", v)),
4447
tracer: tracer,
4548
userService: userService,
4649
phoneService: phoneService,
50+
cache: appCache,
4751
}
4852
}
4953

@@ -79,7 +83,7 @@ func (v *BulkMessageHandlerValidator) ValidateStore(ctx context.Context, userID
7983
messages[index] = message.Sanitize()
8084
}
8185

82-
result = v.validateMessages(messages)
86+
result = v.validateMessages(ctx, messages)
8387
if len(result) != 0 {
8488
return messages, result
8589
}
@@ -215,7 +219,7 @@ func (v *BulkMessageHandlerValidator) parseCSV(ctxLogger telemetry.Logger, user
215219
return messages, url.Values{}
216220
}
217221

218-
func (v *BulkMessageHandlerValidator) validateMessages(messages []*requests.BulkMessage) url.Values {
222+
func (v *BulkMessageHandlerValidator) validateMessages(ctx context.Context, messages []*requests.BulkMessage) url.Values {
219223
result := url.Values{}
220224
for index, message := range messages {
221225

@@ -238,7 +242,7 @@ func (v *BulkMessageHandlerValidator) validateMessages(messages []*requests.Bulk
238242
} else if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
239243
result.Add("document", fmt.Sprintf("Row [%d]: The attachment URL [%s] must use http or https.", index+2, cleanURL))
240244
} else {
241-
if err := validateAttachmentURL(cleanURL); err != nil {
245+
if err := validateAttachmentURL(ctx, v.cache, cleanURL); err != nil {
242246
result.Add("attachments", fmt.Sprintf("Row [%d]: The attachment URL [%s] failed validation: %s", index+2, cleanURL, err.Error()))
243247
}
244248
}

‎api/pkg/validators/message_handler_validator.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strings"
88
"time"
99

10+
"github.com/NdoleStudio/httpsms/pkg/cache"
1011
"github.com/NdoleStudio/httpsms/pkg/repositories"
1112
"github.com/NdoleStudio/httpsms/pkg/services"
1213
"github.com/palantir/stacktrace"
@@ -25,6 +26,7 @@ type MessageHandlerValidator struct {
2526
tracer telemetry.Tracer
2627
phoneService *services.PhoneService
2728
tokenValidator *TurnstileTokenValidator
29+
cache cache.Cache
2830
}
2931

3032
// NewMessageHandlerValidator creates a new handlers.MessageHandler validator
@@ -33,12 +35,14 @@ func NewMessageHandlerValidator(
3335
tracer telemetry.Tracer,
3436
phoneService *services.PhoneService,
3537
tokenValidator *TurnstileTokenValidator,
38+
appCache cache.Cache,
3639
) (v *MessageHandlerValidator) {
3740
return &MessageHandlerValidator{
3841
logger: logger.WithService(fmt.Sprintf("%T", v)),
3942
tracer: tracer,
4043
phoneService: phoneService,
4144
tokenValidator: tokenValidator,
45+
cache: appCache,
4246
}
4347
}
4448

@@ -124,7 +128,7 @@ func (validator MessageHandlerValidator) ValidateMessageSend(ctx context.Context
124128
} else if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
125129
result.Add("attachments", fmt.Sprintf("attachment at index %d must use http or https scheme", i))
126130
} else {
127-
if err := validateAttachmentURL(attachment.URL); err != nil {
131+
if err := validateAttachmentURL(ctx, validator.cache, attachment.URL); err != nil {
128132
result.Add("attachments", fmt.Sprintf("attachment at index %d failed validation: %s", i, err.Error()))
129133
}
130134
}
@@ -199,7 +203,7 @@ func (validator MessageHandlerValidator) ValidateMessageBulkSend(ctx context.Con
199203
} else if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
200204
result.Add("attachments", fmt.Sprintf("attachment at index %d must use http or https scheme", i))
201205
} else {
202-
if err := validateAttachmentURL(attachment.URL); err != nil {
206+
if err := validateAttachmentURL(ctx, validator.cache, attachment.URL); err != nil {
203207
result.Add("attachments", fmt.Sprintf("attachment at index %d failed validation: %s", i, err.Error()))
204208
}
205209
}

‎api/pkg/validators/validator.go‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
package validators
22

33
import (
4+
"context"
45
"fmt"
56
"net/http"
67
"net/url"
78
"regexp"
89
"strings"
910
"time"
1011

12+
"github.com/NdoleStudio/httpsms/pkg/cache"
1113
"github.com/NdoleStudio/httpsms/pkg/events"
1214

1315
"github.com/nyaruka/phonenumbers"
@@ -163,31 +165,53 @@ func (validator *validator) ValidateUUID(ID string, name string) url.Values {
163165
return v.ValidateStruct()
164166
}
165167

166-
func validateAttachmentURL(attachmentURL string) error {
168+
func validateAttachmentURL(ctx context.Context, c cache.Cache, attachmentURL string) error {
169+
cacheKey := "mms-url-validation:" + attachmentURL
170+
171+
if cachedVal, err := c.Get(ctx, cacheKey); err == nil {
172+
if cachedVal == "valid" {
173+
return nil
174+
}
175+
return fmt.Errorf(cachedVal)
176+
}
177+
167178
client := &http.Client{
168179
Timeout: 5 * time.Second,
169180
}
170181

171182
req, err := http.NewRequest(http.MethodHead, attachmentURL, nil)
172183
if err != nil {
173-
return fmt.Errorf("invalid url format")
184+
errMsg := fmt.Sprintf("invalid url format")
185+
saveToCache(ctx, c, cacheKey, errMsg)
186+
return fmt.Errorf(errMsg)
174187
}
175188

176189
resp, err := client.Do(req)
177190
if err != nil {
178-
return fmt.Errorf("could not reach the url")
191+
errMsg := fmt.Sprintf("could not reach the url")
192+
saveToCache(ctx, c, cacheKey, errMsg)
193+
return fmt.Errorf(errMsg)
179194
}
180195
defer resp.Body.Close()
181196

182197
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
183-
return fmt.Errorf("url returned an error status code: %d", resp.StatusCode)
198+
errMsg := fmt.Sprintf("url returned an error status code: %d", resp.StatusCode)
199+
saveToCache(ctx, c, cacheKey, errMsg)
200+
return fmt.Errorf(errMsg)
184201
}
185202

186203
const maxSizeBytes = 1.5 * 1024 * 1024
187204

188205
if resp.ContentLength > int64(maxSizeBytes) {
189-
return fmt.Errorf("file size (%.2f MB) exceeds the 1.5 MB carrier limit", float64(resp.ContentLength)/(1024*1024))
206+
errMsg := fmt.Sprintf("file size (%.2f MB) exceeds the 1.5 MB carrier limit", float64(resp.ContentLength)/(1024*1024))
207+
saveToCache(ctx, c, cacheKey, errMsg)
208+
return fmt.Errorf(errMsg)
190209
}
191210

211+
saveToCache(ctx, c, cacheKey, "valid")
192212
return nil
193213
}
214+
215+
func saveToCache(ctx context.Context, c cache.Cache, key string, value string) {
216+
_ = c.Set(ctx, key, value, 24*time.Hour)
217+
}

0 commit comments

Comments
 (0)