Skip to content

Commit a401ae3

Browse files
timsfeast-ci-bot
tims
authored and
feast-ci-bot
committed
Remove feature specs being able to declare their serving or warehouse stores (#159)
* Ingestion: remove store references from feature specs and expect only a single serving and warehouse storage spec * remove multi store support in core, can no longer register feature but can retreive them * remove multiple serving stores from serving module * fix core tests * fix test for caching serving storage specs * post-rebase fixes * Update CLI to remove references to stores * fix test after merge * pass store spec to serving via env vars * Add list storage functions back in, mark as deprecated * Regenerate mocks * fix go syntax
1 parent 3499b65 commit a401ae3

105 files changed

Lines changed: 2359 additions & 3841 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎charts/feast/templates/serving-deploy.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ spec:
6969
value: "{{ printf "%s.%s.svc.cluster.local" (include "feast.core.name" .) .Release.Namespace }}"
7070
- name: FEAST_CORE_GRPC_PORT
7171
value: "{{ .Values.core.service.grpc.port }}"
72+
- name: STORE_SERVING_TYPE
73+
value: {{ .Values.store.serving.type }}
74+
- name: STORE_SERVING_OPTIONS
75+
value: {{ .Values.store.serving.options | toJson}}
7276
- name: FEAST_MAX_NB_THREAD
7377
value: "{{ .Values.serving.config.maxNumberOfThread }}"
7478
- name: FEAST_MAX_ENTITY_PER_BATCH

‎cli/feast/cmd/apply.go‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,9 @@ Valid resources include:
3737
- entity
3838
- feature
3939
- featureGroup
40-
- storage
4140
4241
Examples:
4342
- feast apply entity entity.yml
44-
- feast apply storage storage1.yml storage2.yml
4543
- feast apply feature *-feature.yml`,
4644
RunE: func(cmd *cobra.Command, args []string) error {
4745
if len(args) == 0 {
@@ -90,8 +88,6 @@ func apply(ctx context.Context, coreCli core.CoreServiceClient, resource string,
9088
return applyFeatureGroup(ctx, coreCli, yml)
9189
case "entity":
9290
return applyEntity(ctx, coreCli, yml)
93-
case "storage":
94-
return applyStorage(ctx, coreCli, yml)
9591
default:
9692
return "", fmt.Errorf("invalid resource %s: please choose one of [feature, featureGroup, entity, storage]", resource)
9793
}
@@ -124,15 +120,6 @@ func applyEntity(ctx context.Context, coreCli core.CoreServiceClient, yml []byte
124120
return es.GetName(), err
125121
}
126122

127-
func applyStorage(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) {
128-
ss, err := parse.YamlToStorageSpec(yml)
129-
if err != nil {
130-
return "", err
131-
}
132-
_, err = coreCli.ApplyStorage(ctx, ss)
133-
return ss.GetId(), err
134-
}
135-
136123
func isYaml(path string) bool {
137124
ext := filepath.Ext(path)
138125
if ext == ".yaml" || ext == ".yml" {

‎cli/feast/cmd/apply_test.go‎

Lines changed: 30 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@ package cmd
1616

1717
import (
1818
"context"
19+
"testing"
20+
1921
"github.com/gojek/feast/cli/feast/cmd/mock"
2022
"github.com/golang/mock/gomock"
21-
"testing"
2223

2324
"github.com/gojek/feast/protos/generated/go/feast/core"
2425
)
@@ -28,7 +29,6 @@ func Test_apply(t *testing.T) {
2829
mockCore.EXPECT().ApplyEntity(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
2930
mockCore.EXPECT().ApplyFeature(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
3031
mockCore.EXPECT().ApplyFeatureGroup(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
31-
mockCore.EXPECT().ApplyStorage(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes()
3232

3333
type args struct {
3434
ctx context.Context
@@ -45,89 +45,78 @@ func Test_apply(t *testing.T) {
4545
{
4646
name: "test apply invalid resource",
4747
args: args{
48-
ctx: context.Background(),
49-
coreCli: mockCore,
50-
resource: "invalidResource",
48+
ctx: context.Background(),
49+
coreCli: mockCore,
50+
resource: "invalidResource",
5151
fileLocation: "testdata/valid_entity.yaml",
5252
},
53-
want: "",
53+
want: "",
5454
wantErr: true,
5555
},
5656
{
5757
name: "test apply entity",
5858
args: args{
59-
ctx: context.Background(),
60-
coreCli: mockCore,
61-
resource: "entity",
59+
ctx: context.Background(),
60+
coreCli: mockCore,
61+
resource: "entity",
6262
fileLocation: "testdata/valid_entity.yaml",
6363
},
64-
want: "myentity",
64+
want: "myentity",
6565
wantErr: false,
6666
},
6767
{
6868
name: "test apply entity with non-existent file",
6969
args: args{
70-
ctx: context.Background(),
71-
coreCli: mockCore,
72-
resource: "entity",
70+
ctx: context.Background(),
71+
coreCli: mockCore,
72+
resource: "entity",
7373
fileLocation: "testdata/file_not_exists.yaml",
7474
},
75-
want: "",
75+
want: "",
7676
wantErr: true,
7777
},
7878
{
7979
name: "test apply entity with no tag",
8080
args: args{
81-
ctx: context.Background(),
82-
coreCli: mockCore,
83-
resource: "entity",
81+
ctx: context.Background(),
82+
coreCli: mockCore,
83+
resource: "entity",
8484
fileLocation: "testdata/valid_entity_no_tag.yaml",
8585
},
86-
want: "myentity",
86+
want: "myentity",
8787
wantErr: false,
8888
},
8989
{
9090
name: "test apply invalid syntax in entity yaml",
9191
args: args{
92-
ctx: context.Background(),
93-
coreCli: mockCore,
94-
resource: "entity",
92+
ctx: context.Background(),
93+
coreCli: mockCore,
94+
resource: "entity",
9595
fileLocation: "testdata/invalid_entity.yaml",
9696
},
97-
want: "",
97+
want: "",
9898
wantErr: true,
9999
},
100100
{
101101
name: "test apply feature",
102102
args: args{
103-
ctx: context.Background(),
104-
coreCli: mockCore,
105-
resource: "feature",
103+
ctx: context.Background(),
104+
coreCli: mockCore,
105+
resource: "feature",
106106
fileLocation: "testdata/valid_feature.yaml",
107107
},
108-
want: "myentity.feature_bool_redis1",
108+
want: "myentity.feature_bool_redis1",
109109
wantErr: false,
110110
},
111111
{
112112
name: "test apply feature group",
113113
args: args{
114-
ctx: context.Background(),
115-
coreCli: mockCore,
116-
resource: "featureGroup",
114+
ctx: context.Background(),
115+
coreCli: mockCore,
116+
resource: "featureGroup",
117117
fileLocation: "testdata/valid_feature_group.yaml",
118118
},
119-
want: "my_fg",
120-
wantErr: false,
121-
},
122-
{
123-
name: "test apply storage",
124-
args: args{
125-
ctx: context.Background(),
126-
coreCli: mockCore,
127-
resource: "storage",
128-
fileLocation: "testdata/valid_storage.yaml",
129-
},
130-
want: "BIGQUERY1",
119+
want: "my_fg",
131120
wantErr: false,
132121
},
133122
}

‎cli/feast/cmd/get.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ var getCmd = &cobra.Command{
1919
Valid resources include:
2020
- entity
2121
- feature
22-
- storage
2322
- job
2423
2524
Examples:
@@ -81,6 +80,7 @@ func getEntity(ctx context.Context, cli core.UIServiceClient, id string) error {
8180
return nil
8281
}
8382

83+
// This function is deprecated, and may be removed in subsequent versions.
8484
func getStorage(ctx context.Context, cli core.UIServiceClient, id string) error {
8585
response, err := cli.GetStorage(ctx, &core.UIServiceTypes_GetStorageRequest{Id: id})
8686
if err != nil {

‎cli/feast/cmd/list.go‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ var listCmd = &cobra.Command{
3838
Valid resources include:
3939
- entities
4040
- features
41-
- storage
4241
- jobs
4342
4443
Examples:
@@ -73,12 +72,12 @@ func list(resource string) error {
7372
return listFeatures(ctx, core.NewCoreServiceClient(coreConn))
7473
case "entities":
7574
return listEntities(ctx, core.NewCoreServiceClient(coreConn))
76-
case "storage":
77-
return listStorage(ctx, core.NewCoreServiceClient(coreConn))
7875
case "jobs":
7976
return listJobs(ctx, core.NewJobServiceClient(coreConn))
77+
case "storage":
78+
return listStorage(ctx, core.NewCoreServiceClient(coreConn))
8079
default:
81-
return fmt.Errorf("invalid resource %s: please choose one of [features, entities, storage, jobs]", resource)
80+
return fmt.Errorf("invalid resource %s: please choose one of [features, storage, entities, jobs]", resource)
8281
}
8382
}
8483

@@ -116,35 +115,36 @@ func listEntities(ctx context.Context, cli core.CoreServiceClient) error {
116115
return nil
117116
}
118117

119-
func listStorage(ctx context.Context, cli core.CoreServiceClient) error {
120-
response, err := cli.ListStorage(ctx, &empty.Empty{})
118+
func listJobs(ctx context.Context, cli core.JobServiceClient) error {
119+
response, err := cli.ListJobs(ctx, &empty.Empty{})
121120
if err != nil {
122121
return err
123122
}
124123
w := new(tabwriter.Writer)
125124
w.Init(os.Stdout, 0, 8, 2, ' ', 0)
126-
fmt.Fprintf(w, "ID\tTYPE\n")
127-
fmt.Fprintf(w, "--\t----\t\n")
128-
for _, feat := range response.GetStorageSpecs() {
125+
fmt.Fprintf(w, "JOB_ID\tTYPE\tRUNNER\tSTATUS\tAGE\n")
126+
fmt.Fprintf(w, "------\t----\t------\t------\t---\n")
127+
for _, job := range response.GetJobs() {
129128
fmt.Fprintf(w, strings.Join(
130-
[]string{feat.Id, feat.Type}, "\t")+"\n")
129+
[]string{job.Id, job.Type, job.Runner, job.Status, timeutil.DurationUntilNowInHumanFormat(*job.Created)}, "\t")+"\n")
131130
}
132131
w.Flush()
133132
return nil
134133
}
135134

136-
func listJobs(ctx context.Context, cli core.JobServiceClient) error {
137-
response, err := cli.ListJobs(ctx, &empty.Empty{})
135+
// This function is deprecated, and may be removed in subsequent versions.
136+
func listStorage(ctx context.Context, cli core.CoreServiceClient) error {
137+
response, err := cli.ListStorage(ctx, &empty.Empty{})
138138
if err != nil {
139139
return err
140140
}
141141
w := new(tabwriter.Writer)
142142
w.Init(os.Stdout, 0, 8, 2, ' ', 0)
143-
fmt.Fprintf(w, "JOB_ID\tTYPE\tRUNNER\tSTATUS\tAGE\n")
144-
fmt.Fprintf(w, "------\t----\t------\t------\t---\n")
145-
for _, job := range response.GetJobs() {
143+
fmt.Fprintf(w, "ID\tTYPE\n")
144+
fmt.Fprintf(w, "--\t----\t\n")
145+
for _, feat := range response.GetStorageSpecs() {
146146
fmt.Fprintf(w, strings.Join(
147-
[]string{job.Id, job.Type, job.Runner, job.Status, timeutil.DurationUntilNowInHumanFormat(*job.Created)}, "\t")+"\n")
147+
[]string{feat.Id, feat.Type}, "\t")+"\n")
148148
}
149149
w.Flush()
150150
return nil

0 commit comments

Comments
 (0)