Skip to content

Commit 58be916

Browse files
pyalexpyalex
authored
Add redis timeout configuration option & Serving Configuration Refactoring (#8)
* add redis timeout option Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * fix typo Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * remove StoreProto.Store Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * remove StoreProto.Store Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * fix test Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * clean up Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
1 parent 5a14e1e commit 58be916

File tree

16 files changed

+128
-494
lines changed

16 files changed

+128
-494
lines changed

‎serving/src/main/java/feast/serving/config/FeastProperties.java‎

Lines changed: 25 additions & 244 deletions
Original file line numberDiff line numberDiff line change
@@ -21,24 +21,18 @@
2121
// https://www.baeldung.com/configuration-properties-in-spring-boot
2222
// https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties
2323

24-
import com.fasterxml.jackson.core.JsonProcessingException;
25-
import com.fasterxml.jackson.databind.ObjectMapper;
26-
import com.google.protobuf.InvalidProtocolBufferException;
27-
import com.google.protobuf.util.JsonFormat;
2824
import feast.common.auth.config.SecurityProperties;
2925
import feast.common.auth.config.SecurityProperties.AuthenticationProperties;
3026
import feast.common.auth.config.SecurityProperties.AuthorizationProperties;
3127
import feast.common.auth.credentials.CoreAuthenticationProperties;
3228
import feast.common.logging.config.LoggingProperties;
33-
import feast.proto.core.StoreProto;
29+
import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig;
30+
import feast.storage.connectors.redis.retriever.RedisStoreConfig;
31+
import io.lettuce.core.ReadFrom;
32+
import java.time.Duration;
3433
import java.util.*;
35-
import java.util.stream.Collectors;
3634
import javax.annotation.PostConstruct;
37-
import javax.validation.ConstraintViolation;
38-
import javax.validation.ConstraintViolationException;
39-
import javax.validation.Validation;
40-
import javax.validation.Validator;
41-
import javax.validation.ValidatorFactory;
35+
import javax.validation.*;
4236
import javax.validation.constraints.NotBlank;
4337
import javax.validation.constraints.NotNull;
4438
import javax.validation.constraints.Positive;
@@ -146,9 +140,6 @@ public void setActiveStore(String activeStore) {
146140
*/
147141
private List<Store> stores = new ArrayList<>();
148142

149-
/* Job Store properties to retain state of async jobs. */
150-
private JobStoreProperties jobStore;
151-
152143
/* Metric tracing properties. */
153144
private TracingProperties tracing;
154145

@@ -259,8 +250,6 @@ public static class Store {
259250

260251
private Map<String, String> config = new HashMap<>();
261252

262-
private List<Subscription> subscriptions = new ArrayList<>();
263-
264253
/**
265254
* Gets name of this store. This is unique to this specific instance.
266255
*
@@ -280,12 +269,12 @@ public void setName(String name) {
280269
}
281270

282271
/**
283-
* Gets the store type. Example are REDIS or BIGQUERY
272+
* Gets the store type. Example are REDIS or REDIS_CLUSTER
284273
*
285274
* @return the store type as a String.
286275
*/
287-
public String getType() {
288-
return type;
276+
public StoreType getType() {
277+
return StoreType.valueOf(this.type);
289278
}
290279

291280
/**
@@ -297,64 +286,6 @@ public void setType(String type) {
297286
this.type = type;
298287
}
299288

300-
/**
301-
* Converts this {@link Store} to a {@link StoreProto.Store}
302-
*
303-
* @return {@link StoreProto.Store} with configuration set
304-
* @throws InvalidProtocolBufferException the invalid protocol buffer exception
305-
* @throws JsonProcessingException the json processing exception
306-
*/
307-
public StoreProto.Store toProto()
308-
throws InvalidProtocolBufferException, JsonProcessingException {
309-
List<Subscription> subscriptions = getSubscriptions();
310-
List<StoreProto.Store.Subscription> subscriptionProtos =
311-
subscriptions.stream().map(Subscription::toProto).collect(Collectors.toList());
312-
313-
StoreProto.Store.Builder storeProtoBuilder =
314-
StoreProto.Store.newBuilder()
315-
.setName(name)
316-
.setType(StoreProto.Store.StoreType.valueOf(type))
317-
.addAllSubscriptions(subscriptionProtos);
318-
319-
ObjectMapper jsonWriter = new ObjectMapper();
320-
321-
// TODO: All of this logic should be moved to the store layer. Only a Map<String, String>
322-
// should be sent to a store and it should do its own validation.
323-
switch (StoreProto.Store.StoreType.valueOf(type)) {
324-
case REDIS_CLUSTER:
325-
StoreProto.Store.RedisClusterConfig.Builder redisClusterConfig =
326-
StoreProto.Store.RedisClusterConfig.newBuilder();
327-
JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisClusterConfig);
328-
return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig.build()).build();
329-
case REDIS:
330-
StoreProto.Store.RedisConfig.Builder redisConfig =
331-
StoreProto.Store.RedisConfig.newBuilder();
332-
JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisConfig);
333-
return storeProtoBuilder.setRedisConfig(redisConfig.build()).build();
334-
default:
335-
throw new InvalidProtocolBufferException("Invalid store set");
336-
}
337-
}
338-
339-
/**
340-
* Get the subscriptions to this specific store. The subscriptions indicate which feature sets a
341-
* store subscribes to.
342-
*
343-
* @return List of subscriptions.
344-
*/
345-
public List<Subscription> getSubscriptions() {
346-
return subscriptions;
347-
}
348-
349-
/**
350-
* Sets the store specific configuration. See getSubscriptions() for more details.
351-
*
352-
* @param subscriptions the subscriptions list
353-
*/
354-
public void setSubscriptions(List<Subscription> subscriptions) {
355-
this.subscriptions = subscriptions;
356-
}
357-
358289
/**
359290
* Gets the configuration to this specific store. This is a map of strings. These options are
360291
* unique to the store. Please see protos/feast/core/Store.proto for the store specific
@@ -366,6 +297,20 @@ public Map<String, String> getConfig() {
366297
return config;
367298
}
368299

300+
public RedisClusterStoreConfig getRedisClusterConfig() {
301+
return new RedisClusterStoreConfig(
302+
this.config.get("connection_string"),
303+
ReadFrom.valueOf(this.config.get("read_from")),
304+
Duration.parse(this.config.get("timeout")));
305+
}
306+
307+
public RedisStoreConfig getRedisConfig() {
308+
return new RedisStoreConfig(
309+
this.config.get("host"),
310+
Integer.valueOf(this.config.get("port")),
311+
Boolean.valueOf(this.config.getOrDefault("ssl", "false")));
312+
}
313+
369314
/**
370315
* Sets the store config. Please protos/feast/core/Store.proto for the specific options for each
371316
* store.
@@ -375,129 +320,11 @@ public Map<String, String> getConfig() {
375320
public void setConfig(Map<String, String> config) {
376321
this.config = config;
377322
}
378-
379-
/**
380-
* The Subscription type.
381-
*
382-
* <p>Note: Please see protos/feast/core/CoreService.proto for details on how to subscribe to
383-
* feature sets.
384-
*/
385-
public static class Subscription {
386-
/** Feast project to subscribe to. */
387-
String project;
388-
389-
/** Feature set to subscribe to. */
390-
String name;
391-
392-
/** Feature set versions to subscribe to. */
393-
String version;
394-
395-
/** Project/Feature set exclude flag to subscribe to. */
396-
boolean exclude;
397-
398-
/**
399-
* Gets Feast project subscribed to.
400-
*
401-
* @return the project string
402-
*/
403-
public String getProject() {
404-
return project;
405-
}
406-
407-
/**
408-
* Sets Feast project to subscribe to for this store.
409-
*
410-
* @param project the project
411-
*/
412-
public void setProject(String project) {
413-
this.project = project;
414-
}
415-
416-
/**
417-
* Gets the feature set name to subscribe to.
418-
*
419-
* @return the name
420-
*/
421-
public String getName() {
422-
return name;
423-
}
424-
425-
/**
426-
* Sets the feature set name to subscribe to.
427-
*
428-
* @param name the name
429-
*/
430-
public void setName(String name) {
431-
this.name = name;
432-
}
433-
434-
/**
435-
* Gets the feature set version that is being subscribed to by this store.
436-
*
437-
* @return the version
438-
*/
439-
public String getVersion() {
440-
return version;
441-
}
442-
443-
/**
444-
* Sets the feature set version that is being subscribed to by this store.
445-
*
446-
* @param version the version
447-
*/
448-
public void setVersion(String version) {
449-
this.version = version;
450-
}
451-
452-
/**
453-
* Gets the exclude flag to subscribe to.
454-
*
455-
* @return the exclude flag
456-
*/
457-
public boolean getExclude() {
458-
return exclude;
459-
}
460-
461-
/**
462-
* Sets the exclude flag to subscribe to.
463-
*
464-
* @param exclude the exclude flag
465-
*/
466-
public void setExclude(boolean exclude) {
467-
this.exclude = exclude;
468-
}
469-
470-
/**
471-
* Convert this {@link Subscription} to a {@link StoreProto.Store.Subscription}.
472-
*
473-
* @return the store proto . store . subscription
474-
*/
475-
public StoreProto.Store.Subscription toProto() {
476-
return StoreProto.Store.Subscription.newBuilder()
477-
.setName(getName())
478-
.setProject(getProject())
479-
.setExclude(getExclude())
480-
.build();
481-
}
482-
}
483323
}
484324

485-
/**
486-
* Gets job store properties
487-
*
488-
* @return the job store properties
489-
*/
490-
public JobStoreProperties getJobStore() {
491-
return jobStore;
492-
}
493-
494-
/**
495-
* Set job store properties
496-
*
497-
* @param jobStore Job store properties to set
498-
*/
499-
public void setJobStore(JobStoreProperties jobStore) {
500-
this.jobStore = jobStore;
325+
public enum StoreType {
326+
REDIS,
327+
REDIS_CLUSTER;
501328
}
502329

503330
/**
@@ -532,52 +359,6 @@ public void setLogging(LoggingProperties logging) {
532359
this.logging = logging;
533360
}
534361

535-
/** The type Job store properties. */
536-
public static class JobStoreProperties {
537-
538-
/** Job Store Redis Host */
539-
private String redisHost;
540-
541-
/** Job Store Redis Host */
542-
private int redisPort;
543-
544-
/**
545-
* Gets redis host.
546-
*
547-
* @return the redis host
548-
*/
549-
public String getRedisHost() {
550-
return redisHost;
551-
}
552-
553-
/**
554-
* Sets redis host.
555-
*
556-
* @param redisHost the redis host
557-
*/
558-
public void setRedisHost(String redisHost) {
559-
this.redisHost = redisHost;
560-
}
561-
562-
/**
563-
* Gets redis port.
564-
*
565-
* @return the redis port
566-
*/
567-
public int getRedisPort() {
568-
return redisPort;
569-
}
570-
571-
/**
572-
* Sets redis port.
573-
*
574-
* @param redisPort the redis port
575-
*/
576-
public void setRedisPort(int redisPort) {
577-
this.redisPort = redisPort;
578-
}
579-
}
580-
581362
/** Trace metric collection properties */
582363
public static class TracingProperties {
583364

‎serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
import com.fasterxml.jackson.core.JsonProcessingException;
2020
import com.google.protobuf.InvalidProtocolBufferException;
21-
import feast.proto.core.StoreProto;
2221
import feast.serving.service.OnlineServingServiceV2;
2322
import feast.serving.service.ServingServiceV2;
2423
import feast.serving.specs.CachedSpecService;
@@ -39,26 +38,19 @@ public ServingServiceV2 servingServiceV2(
3938
throws InvalidProtocolBufferException, JsonProcessingException {
4039
ServingServiceV2 servingService = null;
4140
FeastProperties.Store store = feastProperties.getActiveStore();
42-
StoreProto.Store.StoreType storeType = store.toProto().getType();
4341

44-
switch (storeType) {
42+
switch (store.getType()) {
4543
case REDIS_CLUSTER:
4644
RedisClientAdapter redisClusterClient =
47-
RedisClusterClient.create(store.toProto().getRedisClusterConfig());
45+
RedisClusterClient.create(store.getRedisClusterConfig());
4846
OnlineRetrieverV2 redisClusterRetriever = new OnlineRetriever(redisClusterClient);
4947
servingService = new OnlineServingServiceV2(redisClusterRetriever, specService, tracer);
5048
break;
5149
case REDIS:
52-
RedisClientAdapter redisClient = RedisClient.create(store.toProto().getRedisConfig());
50+
RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig());
5351
OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient);
5452
servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer);
5553
break;
56-
case UNRECOGNIZED:
57-
case INVALID:
58-
throw new IllegalArgumentException(
59-
String.format(
60-
"Unsupported store type '%s' for store name '%s'",
61-
store.getType(), store.getName()));
6254
}
6355

6456
return servingService;

‎serving/src/main/java/feast/serving/config/SpecServiceConfig.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
import com.fasterxml.jackson.core.JsonProcessingException;
2020
import com.google.protobuf.InvalidProtocolBufferException;
21-
import feast.proto.core.StoreProto;
2221
import feast.serving.specs.CachedSpecService;
2322
import feast.serving.specs.CoreSpecService;
2423
import io.grpc.CallCredentials;
@@ -61,13 +60,11 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService(
6160
}
6261

6362
@Bean
64-
public CachedSpecService specService(
65-
FeastProperties feastProperties, ObjectProvider<CallCredentials> callCredentials)
63+
public CachedSpecService specService(ObjectProvider<CallCredentials> callCredentials)
6664
throws InvalidProtocolBufferException, JsonProcessingException {
6765
CoreSpecService coreService =
6866
new CoreSpecService(feastCoreHost, feastCorePort, callCredentials);
69-
StoreProto.Store storeProto = feastProperties.getActiveStore().toProto();
70-
CachedSpecService cachedSpecStorage = new CachedSpecService(coreService, storeProto);
67+
CachedSpecService cachedSpecStorage = new CachedSpecService(coreService);
7168
try {
7269
cachedSpecStorage.populateCache();
7370
} catch (Exception e) {

0 commit comments

Comments
 (0)