Skip to content

Commit f60155a

Browse files
terryyylimkhorshuheng
terryyylim
and
khorshuheng
authored
Add support for Cassandra Online Storage (#21)
* Add cassandra storage module and scaffold IT Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Add partial implementation of Cassandra retriever Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Add partial implementation Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Update retrieval query and create schema table in IT Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Fix IT Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Retrieve timestamp and schema ref based on feature table Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Fix retrieval logic Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Refactor IT Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Add more tests Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Fix retrieval logic for same entity with multiple featuretable Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Add IT for same entity with multiple featuretable retrieval Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Use connection string instead of single host port pair Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Update application.yml configuration Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Update tests Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Fix schema caching for Cassandra Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Fix formatting Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Refactor common functionality Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Use default interface instead of concrete base class for sstable based retrievers Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Extract common functionalities for bigttable and cassandra retriever Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Fix formatting Signed-off-by: Terence Lim <terencelimxp@gmail.com> * Ignore features with null values Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> Co-authored-by: Khor Shu Heng <khor.heng@gojek.com>
1 parent bea6f21 commit f60155a

File tree

17 files changed

+1523
-164
lines changed

17 files changed

+1523
-164
lines changed

‎common-test/src/main/java/feast/common/it/DataGenerator.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,10 @@ public static ValueProto.Value createDoubleValue(double value) {
249249
return ValueProto.Value.newBuilder().setDoubleVal(value).build();
250250
}
251251

252+
public static ValueProto.Value createInt32Value(int value) {
253+
return ValueProto.Value.newBuilder().setInt32Val(value).build();
254+
}
255+
252256
public static ValueProto.Value createInt64Value(long value) {
253257
return ValueProto.Value.newBuilder().setInt64Val(value).build();
254258
}

‎serving/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,12 @@
9090
<version>${project.version}</version>
9191
</dependency>
9292

93+
<dependency>
94+
<groupId>dev.feast</groupId>
95+
<artifactId>feast-storage-connector-cassandra</artifactId>
96+
<version>${project.version}</version>
97+
</dependency>
98+
9399
<dependency>
94100
<groupId>dev.feast</groupId>
95101
<artifactId>feast-common</artifactId>

‎serving/src/main/java/feast/serving/config/FeastProperties.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import feast.common.auth.credentials.CoreAuthenticationProperties;
2828
import feast.common.logging.config.LoggingProperties;
2929
import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig;
30+
import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig;
3031
import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig;
3132
import feast.storage.connectors.redis.retriever.RedisStoreConfig;
3233
import io.lettuce.core.ReadFrom;
@@ -270,7 +271,7 @@ public void setName(String name) {
270271
}
271272

272273
/**
273-
* Gets the store type. Example are REDIS, REDIS_CLUSTER or BIGTABLE
274+
* Gets the store type. Example are REDIS, REDIS_CLUSTER, BIGTABLE or CASSANDRA
274275
*
275276
* @return the store type as a String.
276277
*/
@@ -316,6 +317,13 @@ public BigTableStoreConfig getBigtableConfig() {
316317
return new BigTableStoreConfig(this.config.get("project_id"), this.config.get("instance_id"));
317318
}
318319

320+
public CassandraStoreConfig getCassandraConfig() {
321+
return new CassandraStoreConfig(
322+
this.config.get("connection_string"),
323+
this.config.get("data_center"),
324+
this.config.get("keyspace"));
325+
}
326+
319327
/**
320328
* Sets the store config. Please protos/feast/core/Store.proto for the specific options for each
321329
* store.
@@ -329,6 +337,7 @@ public void setConfig(Map<String, String> config) {
329337

330338
public enum StoreType {
331339
BIGTABLE,
340+
CASSANDRA,
332341
REDIS,
333342
REDIS_CLUSTER;
334343
}

‎serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
*/
1717
package feast.serving.config;
1818

19+
import com.datastax.oss.driver.api.core.CqlSession;
20+
import com.datastax.oss.driver.api.core.CqlSessionBuilder;
1921
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
2022
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
2123
import feast.serving.service.OnlineServingServiceV2;
@@ -24,9 +26,15 @@
2426
import feast.storage.api.retriever.OnlineRetrieverV2;
2527
import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever;
2628
import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig;
29+
import feast.storage.connectors.cassandra.retriever.CassandraOnlineRetriever;
30+
import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig;
2731
import feast.storage.connectors.redis.retriever.*;
2832
import io.opentracing.Tracer;
2933
import java.io.IOException;
34+
import java.net.InetSocketAddress;
35+
import java.util.Arrays;
36+
import java.util.List;
37+
import java.util.stream.Collectors;
3038
import org.slf4j.Logger;
3139
import org.springframework.beans.factory.annotation.Autowired;
3240
import org.springframework.context.ApplicationContext;
@@ -77,6 +85,32 @@ public ServingServiceV2 servingServiceV2(
7785
OnlineRetrieverV2 bigtableRetriever = new BigTableOnlineRetriever(bigtableClient);
7886
servingService = new OnlineServingServiceV2(bigtableRetriever, specService, tracer);
7987
break;
88+
case CASSANDRA:
89+
CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig();
90+
String connectionString = config.getConnectionString();
91+
String dataCenter = config.getDataCenter();
92+
String keySpace = config.getKeySpace();
93+
94+
List<InetSocketAddress> contactPoints =
95+
Arrays.stream(connectionString.split(","))
96+
.map(String::trim)
97+
.map(cs -> cs.split(":"))
98+
.map(
99+
hostPort -> {
100+
int port = hostPort.length > 1 ? Integer.parseInt(hostPort[1]) : 9042;
101+
return new InetSocketAddress(hostPort[0], port);
102+
})
103+
.collect(Collectors.toList());
104+
105+
CqlSession session =
106+
new CqlSessionBuilder()
107+
.addContactPoints(contactPoints)
108+
.withLocalDatacenter(dataCenter)
109+
.withKeyspace(keySpace)
110+
.build();
111+
OnlineRetrieverV2 cassandraRetriever = new CassandraOnlineRetriever(session);
112+
servingService = new OnlineServingServiceV2(cassandraRetriever, specService, tracer);
113+
break;
80114
}
81115

82116
return servingService;

‎serving/src/main/resources/application.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ feast:
5959
config:
6060
project_id: <gcp_project>
6161
instance_id: <gcp_bigtable_instance>
62+
- name: cassandra
63+
type: CASSANDRA
64+
config:
65+
connection_string: localhost:9042
66+
data_center: datacenter1
67+
keyspace: feast
6268
tracing:
6369
# If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls
6470
# which can be useful to debug performance issues and perform benchmarking

‎serving/src/test/java/feast/serving/it/BaseAuthIT.java‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@ public class BaseAuthIT {
5454
static final String BIGTABLE = "bigtable_1";
5555
static final int BIGTABLE_PORT = 8086;
5656

57+
static final String CASSANDRA = "cassandra_1";
58+
static final int CASSANDRA_PORT = 9042;
59+
static final String CASSANDRA_DATACENTER = "datacenter1";
60+
static final String CASSANDRA_KEYSPACE = "feast";
61+
static final String CASSANDRA_SCHEMA_TABLE = "feast_schema_reference";
62+
static final String CASSANDRA_ENTITY_KEY = "key";
63+
5764
static final int FEAST_CORE_PORT = 6565;
5865

5966
@DynamicPropertySource
@@ -72,14 +79,40 @@ static void properties(DynamicPropertyRegistry registry) {
7279
}
7380
});
7481
registry.add("feast.stores[0].config.port", () -> REDIS_PORT);
75-
registry.add("feast.stores[0].subscriptions[0].name", () -> "*");
76-
registry.add("feast.stores[0].subscriptions[0].project", () -> "*");
7782

7883
registry.add("feast.stores[1].name", () -> "bigtable");
7984
registry.add("feast.stores[1].type", () -> "BIGTABLE");
8085
registry.add("feast.stores[1].config.project_id", () -> "test-project");
8186
registry.add("feast.stores[1].config.instance_id", () -> "test-instance");
8287

88+
registry.add("feast.stores[2].name", () -> "cassandra");
89+
registry.add("feast.stores[2].type", () -> "CASSANDRA");
90+
registry.add(
91+
"feast.stores[2].config.host",
92+
() -> {
93+
try {
94+
return InetAddress.getLocalHost().getHostAddress();
95+
} catch (UnknownHostException e) {
96+
e.printStackTrace();
97+
return "";
98+
}
99+
});
100+
101+
registry.add(
102+
"feast.stores[2].config.connection_string",
103+
() -> {
104+
String hostAddress = "";
105+
try {
106+
hostAddress = InetAddress.getLocalHost().getHostAddress();
107+
} catch (UnknownHostException e) {
108+
e.printStackTrace();
109+
}
110+
111+
return String.format("%s:%s", hostAddress, CASSANDRA_PORT);
112+
});
113+
registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER);
114+
registry.add("feast.stores[2].config.keyspace", () -> CASSANDRA_KEYSPACE);
115+
83116
registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL);
84117
registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE);
85118
registry.add("feast.core-authentication.options.client_id", () -> CLIENT_ID);

0 commit comments

Comments
 (0)