forked from feast-dev/feast
-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathStore.proto
More file actions
173 lines (155 loc) · 6.39 KB
/
Store.proto
File metadata and controls
173 lines (155 loc) · 6.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
//
// * Copyright 2019 The Feast Authors
// *
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// *
// * https://www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
//
syntax = "proto3";
package feast.core;
option java_package = "feast.core";
option java_outer_classname = "StoreProto";
option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core";
// Store provides a location where Feast reads and writes feature values.
// Feature values will be written to the Store in the form of FeatureRow elements.
// The way FeatureRow is encoded and decoded when it is written to and read from
// the Store depends on the type of the Store.
//
// For example, a FeatureRow will materialize as a row in a table in
// BigQuery but it will materialize as a key, value pair element in Redis.
//
message Store {
enum StoreType {
INVALID = 0;
// Redis stores a FeatureRow element as a key, value pair.
//
// The Redis data types used (https://redis.io/topics/data-types):
// - key: STRING
// - value: STRING
//
// Encodings:
// - key: byte array of RedisKey (refer to feast.storage.RedisKey)
// - value: byte array of FeatureRow (refer to feast.types.FeatureRow)
//
REDIS = 1;
// BigQuery stores a FeatureRow element as a row in a BigQuery table.
//
// Table name is derived from the feature set name and version as:
// [feature_set_name]_v[feature_set_version]
//
// For example:
// A feature row for feature set "driver" and version "1" will be written
// to table "driver_v1".
//
// The entities and features in a FeatureSetSpec corresponds to the
// fields in the BigQuery table (these make up the BigQuery schema).
// The name of the entity spec and feature spec corresponds to the column
// names, and the value_type of entity spec and feature spec corresponds
// to BigQuery standard SQL data type of the column.
//
// The following BigQuery fields are reserved for Feast internal use.
// Ingestion of entity or feature spec with names identical
// to the following field names will raise an exception during ingestion.
//
// column_name | column_data_type | description
// ====================|==================|================================
// - event_timestamp | TIMESTAMP | event time of the FeatureRow
// - created_timestamp | TIMESTAMP | processing time of the ingestion of the FeatureRow
// - job_id | STRING | identifier for the job that writes the FeatureRow to the corresponding BigQuery table
//
// BigQuery table created will be partitioned by the field "event_timestamp"
// of the FeatureRow (https://cloud.google.com/bigquery/docs/partitioned-tables).
//
// Since newer version of feature set can introduce breaking, non backward-
// compatible BigQuery schema updates, incrementing the version of a
// feature set will result in the creation of a new empty BigQuery table
// with the new schema.
//
// The following table shows how ValueType in Feast is mapped to
// BigQuery Standard SQL data types
// (https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types):
//
// BYTES : BYTES
// STRING : STRING
// INT32 : INT64
// INT64 : IN64
// DOUBLE : FLOAT64
// FLOAT : FLOAT64
// BOOL : BOOL
// BYTES_LIST : ARRAY
// STRING_LIST : ARRAY
// INT32_LIST : ARRAY
// INT64_LIST : ARRAY
// DOUBLE_LIST : ARRAY
// FLOAT_LIST : ARRAY
// BOOL_LIST : ARRAY
//
// The column mode in BigQuery is set to "Nullable" such that unset Value
// in a FeatureRow corresponds to NULL value in BigQuery.
//
BIGQUERY = 2;
// Unsupported in Feast 0.3
CASSANDRA = 3;
}
message RedisConfig {
string host = 1;
int32 port = 2;
// Optional. The number of milliseconds to wait before retrying failed Redis connection.
// By default, Feast uses exponential backoff policy and "initial_backoff_ms" sets the initial wait duration.
int32 initial_backoff_ms = 3;
// Optional. Maximum total number of retries for connecting to Redis. Default to zero retries.
int32 max_retries = 4;
}
message BigQueryConfig {
string project_id = 1;
string dataset_id = 2;
}
message CassandraConfig {
string host = 1;
int32 port = 2;
}
message Subscription {
// Name of project that the feature sets belongs to. This can be one of
// - [project_name]
// - *
// If an asterisk is provided, filtering on projects will be disabled. All projects will
// be matched. It is NOT possible to provide an asterisk with a string in order to do
// pattern matching.
string project = 3;
// Name of the desired feature set. Asterisks can be used as wildcards in the name.
// Matching on names is only permitted if a specific project is defined. It is disallowed
// If the project name is set to "*"
// e.g.
// - * can be used to match all feature sets
// - my-feature-set* can be used to match all features prefixed by "my-feature-set"
// - my-feature-set-6 can be used to select a single feature set
string name = 1;
// Versions of the given feature sets that will be returned.
// Valid options for version:
// "latest": only the latest version is returned.
// "*": Subscribe to all versions
// [version number]: pin to a specific version. Project and feature set name must be
// explicitly defined if a specific version is pinned.
string version = 2;
}
// Name of the store.
string name = 1;
// Type of store.
StoreType type = 2;
// Feature sets to subscribe to.
repeated Subscription subscriptions = 4;
// Configuration to connect to the store. Required.
oneof config {
RedisConfig redis_config = 11;
BigQueryConfig bigquery_config = 12;
CassandraConfig cassandra_config = 13;
}
}