Skip to content

Commit dad7e6e

Browse files
timsfeast-ci-bot
tims
authored and
feast-ci-bot
committed
Refactor stores (#110)
* rename feature store classes to feature store factories * split stores up into serving and warehouse and rename some things * rename FeatureIO.Write to FeatureStoreWrite
1 parent 6cdd173 commit dad7e6e

File tree

68 files changed

+659
-721
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+659
-721
lines changed

‎ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@
2828
import feast.ingestion.service.SpecService.Builder;
2929
import feast.ingestion.service.SpecService.UnsupportedBuilder;
3030
import feast.specs.ImportSpecProto.ImportSpec;
31-
import feast.storage.ErrorsStore;
32-
import feast.storage.ServingStore;
33-
import feast.storage.WarehouseStore;
34-
import feast.storage.service.ErrorsStoreService;
35-
import feast.storage.service.ServingStoreService;
36-
import feast.storage.service.WarehouseStoreService;
31+
import feast.store.errors.FeatureErrorsFactory;
32+
import feast.store.errors.FeatureErrorsFactoryService;
33+
import feast.store.serving.FeatureServingFactory;
34+
import feast.store.serving.FeatureServingFactoryService;
35+
import feast.store.warehouse.FeatureWarehouseFactory;
36+
import feast.store.warehouse.FeatureWarehouseFactoryService;
3737
import java.util.List;
3838
import org.apache.beam.sdk.options.PipelineOptions;
3939

@@ -78,19 +78,19 @@ Specs provideSpecs(SpecService.Builder specService) {
7878

7979
@Provides
8080
@Singleton
81-
List<WarehouseStore> provideWarehouseStores() {
82-
return WarehouseStoreService.getAll();
81+
List<FeatureWarehouseFactory> provideWarehouseStores() {
82+
return FeatureWarehouseFactoryService.getAll();
8383
}
8484

8585
@Provides
8686
@Singleton
87-
List<ServingStore> provideServingStores() {
88-
return ServingStoreService.getAll();
87+
List<FeatureServingFactory> provideServingStores() {
88+
return FeatureServingFactoryService.getAll();
8989
}
9090

9191
@Provides
9292
@Singleton
93-
List<ErrorsStore> provideErrorsStores() {
94-
return ErrorsStoreService.getAll();
93+
List<FeatureErrorsFactory> provideErrorsStores() {
94+
return FeatureErrorsFactoryService.getAll();
9595
}
9696
}

‎ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@
2323
import com.google.inject.Inject;
2424
import feast.ingestion.model.Specs;
2525
import feast.ingestion.options.ImportJobPipelineOptions;
26-
import feast.ingestion.transform.FeatureIO.Write;
2726
import feast.specs.StorageSpecProto.StorageSpec;
28-
import feast.storage.ErrorsStore;
29-
import feast.storage.noop.NoOpIO;
27+
import feast.store.FeatureStoreWrite;
28+
import feast.store.NoOpIO;
29+
import feast.store.errors.FeatureErrorsFactory;
3030
import feast.types.FeatureRowExtendedProto.FeatureRowExtended;
3131
import java.util.List;
3232
import lombok.extern.slf4j.Slf4j;
@@ -35,18 +35,19 @@
3535
import org.apache.hadoop.hbase.util.Strings;
3636

3737
@Slf4j
38-
public class ErrorsStoreTransform extends FeatureIO.Write {
38+
public class ErrorsStoreTransform extends FeatureStoreWrite {
3939

4040
private String errorsStoreType;
4141
private StorageSpec errorsStoreSpec;
4242
private Specs specs;
43-
private List<ErrorsStore> errorsStores;
43+
private List<FeatureErrorsFactory> errorsStoreFactories;
4444

4545
@Inject
4646
public ErrorsStoreTransform(
47-
ImportJobPipelineOptions options, Specs specs, List<ErrorsStore> errorsStores) {
47+
ImportJobPipelineOptions options, Specs specs,
48+
List<FeatureErrorsFactory> errorsStoreFactories) {
4849
this.specs = specs;
49-
this.errorsStores = errorsStores;
50+
this.errorsStoreFactories = errorsStoreFactories;
5051
this.errorsStoreType = options.getErrorsStoreType();
5152

5253
if (!Strings.isEmpty(errorsStoreType)) {
@@ -60,7 +61,7 @@ public ErrorsStoreTransform(
6061

6162
@Override
6263
public PDone expand(PCollection<FeatureRowExtended> input) {
63-
Write write;
64+
FeatureStoreWrite write;
6465
if (Strings.isEmpty(errorsStoreType)) {
6566
write = new NoOpIO.Write();
6667
} else {
@@ -70,11 +71,11 @@ public PDone expand(PCollection<FeatureRowExtended> input) {
7071
return PDone.in(input.getPipeline());
7172
}
7273

73-
ErrorsStore getErrorStore() {
74+
FeatureErrorsFactory getErrorStore() {
7475
checkArgument(!errorsStoreType.isEmpty(), "Errors store type not provided");
75-
for (ErrorsStore errorsStore : errorsStores) {
76-
if (errorsStore.getType().equals(errorsStoreType)) {
77-
return errorsStore;
76+
for (FeatureErrorsFactory errorsStoreFactory : errorsStoreFactories) {
77+
if (errorsStoreFactory.getType().equals(errorsStoreType)) {
78+
return errorsStoreFactory;
7879
}
7980
}
8081
throw new IllegalArgumentException("Errors store type not found");

‎ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,20 @@
2121
import feast.ingestion.metrics.FeastMetrics;
2222
import feast.ingestion.model.Specs;
2323
import feast.ingestion.values.PFeatureRows;
24-
import feast.storage.ServingStore;
24+
import feast.store.serving.FeatureServingFactory;
25+
import java.util.List;
2526
import lombok.extern.slf4j.Slf4j;
2627
import org.apache.beam.sdk.transforms.PTransform;
2728
import org.apache.beam.sdk.transforms.ParDo;
2829

29-
import java.util.List;
30-
3130
@Slf4j
3231
public class ServingStoreTransform extends PTransform<PFeatureRows, PFeatureRows> {
3332

34-
private List<ServingStore> stores;
33+
private List<FeatureServingFactory> stores;
3534
private Specs specs;
3635

3736
@Inject
38-
public ServingStoreTransform(List<ServingStore> stores, Specs specs) {
37+
public ServingStoreTransform(List<FeatureServingFactory> stores, Specs specs) {
3938
this.stores = stores;
4039
this.specs = specs;
4140
}
@@ -50,7 +49,8 @@ public PFeatureRows expand(PFeatureRows input) {
5049

5150
output.getMain().apply("metrics.store.lag", ParDo.of(FeastMetrics.lagUpdateDoFn()));
5251
output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("serving_stored")));
53-
output.getErrors().apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("serving_errors")));
52+
output.getErrors()
53+
.apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("serving_errors")));
5454
return output;
5555
}
5656
}

‎ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@
2020
import com.google.common.base.Preconditions;
2121
import com.google.common.collect.Lists;
2222
import feast.ingestion.model.Specs;
23-
import feast.ingestion.transform.FeatureIO.Write;
2423
import feast.ingestion.transform.SplitFeatures.MultiOutputSplit;
2524
import feast.ingestion.values.PFeatureRows;
2625
import feast.specs.FeatureSpecProto.FeatureSpec;
2726
import feast.specs.StorageSpecProto.StorageSpec;
28-
import feast.storage.FeatureStore;
27+
import feast.store.FeatureStoreWrite;
28+
import feast.store.FeatureStoreFactory;
2929
import feast.types.FeatureRowExtendedProto.FeatureRowExtended;
3030
import java.util.Collection;
3131
import java.util.HashMap;
@@ -46,20 +46,20 @@
4646
@Slf4j
4747
public class SplitOutputByStore extends PTransform<PFeatureRows, PFeatureRows> {
4848

49-
private Collection<? extends FeatureStore> stores;
49+
private Collection<? extends FeatureStoreFactory> stores;
5050
private SerializableFunction<FeatureSpec, String> selector;
5151
private Specs specs;
5252

5353
@Override
5454
public PFeatureRows expand(PFeatureRows input) {
55-
Map<String, Write> transforms = getFeatureStoreTransforms();
55+
Map<String, FeatureStoreWrite> transforms = getFeatureStoreTransforms();
5656
Set<String> keys = transforms.keySet();
5757

5858
log.info(String.format("Splitting on keys = [%s]", String.join(",", keys)));
5959
MultiOutputSplit<String> splitter = new MultiOutputSplit<>(selector, keys, specs);
6060
PCollectionTuple splits = input.getMain().apply(splitter);
6161

62-
Map<TupleTag<FeatureRowExtended>, Write> taggedTransforms = new HashMap<>();
62+
Map<TupleTag<FeatureRowExtended>, FeatureStoreWrite> taggedTransforms = new HashMap<>();
6363
for (String key : transforms.keySet()) {
6464
TupleTag<FeatureRowExtended> tag = splitter.getStrategy().getTag(key);
6565
taggedTransforms.put(tag, transforms.get(key));
@@ -72,17 +72,17 @@ public PFeatureRows expand(PFeatureRows input) {
7272
input.getErrors());
7373
}
7474

75-
private Map<String, FeatureStore> getStoresMap() {
76-
Map<String, FeatureStore> storesMap = new HashMap<>();
77-
for (FeatureStore servingStore : stores) {
75+
private Map<String, FeatureStoreFactory> getStoresMap() {
76+
Map<String, FeatureStoreFactory> storesMap = new HashMap<>();
77+
for (FeatureStoreFactory servingStore : stores) {
7878
storesMap.put(servingStore.getType(), servingStore);
7979
}
8080
return storesMap;
8181
}
8282

83-
private Map<String, Write> getFeatureStoreTransforms() {
84-
Map<String, FeatureStore> storesMap = getStoresMap();
85-
Map<String, Write> transforms = new HashMap<>();
83+
private Map<String, FeatureStoreWrite> getFeatureStoreTransforms() {
84+
Map<String, FeatureStoreFactory> storesMap = getStoresMap();
85+
Map<String, FeatureStoreWrite> transforms = new HashMap<>();
8686
Map<String, StorageSpec> storageSpecs = specs.getStorageSpecs();
8787
for (String storeId : storageSpecs.keySet()) {
8888
StorageSpec storageSpec = storageSpecs.get(storeId);
@@ -109,14 +109,14 @@ private Map<String, Write> getFeatureStoreTransforms() {
109109
public static class WriteTags extends
110110
PTransform<PCollectionTuple, PCollection<FeatureRowExtended>> {
111111

112-
private Map<TupleTag<FeatureRowExtended>, Write> transforms;
112+
private Map<TupleTag<FeatureRowExtended>, FeatureStoreWrite> transforms;
113113
private TupleTag<FeatureRowExtended> mainTag;
114114

115115
@Override
116116
public PCollection<FeatureRowExtended> expand(PCollectionTuple tuple) {
117117
List<PCollection<FeatureRowExtended>> outputList = Lists.newArrayList();
118118
for (TupleTag<FeatureRowExtended> tag : transforms.keySet()) {
119-
Write write = transforms.get(tag);
119+
FeatureStoreWrite write = transforms.get(tag);
120120
Preconditions.checkNotNull(write, String.format("Null transform for tag=%s", tag.getId()));
121121
PCollection<FeatureRowExtended> input = tuple.get(tag);
122122
input.apply(String.format("Write to %s", tag.getId()), write);

‎ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,23 @@
1818
package feast.ingestion.transform;
1919

2020
import com.google.inject.Inject;
21+
import feast.ingestion.metrics.FeastMetrics;
22+
import feast.ingestion.model.Specs;
23+
import feast.ingestion.values.PFeatureRows;
24+
import feast.store.warehouse.FeatureWarehouseFactory;
2125
import java.util.List;
2226
import lombok.extern.slf4j.Slf4j;
2327
import org.apache.beam.sdk.transforms.PTransform;
2428
import org.apache.beam.sdk.transforms.ParDo;
25-
import feast.ingestion.metrics.FeastMetrics;
26-
import feast.ingestion.model.Specs;
27-
import feast.ingestion.values.PFeatureRows;
28-
import feast.storage.WarehouseStore;
2929

3030
@Slf4j
3131
public class WarehouseStoreTransform extends PTransform<PFeatureRows, PFeatureRows> {
3232

33-
private List<WarehouseStore> stores;
33+
private List<FeatureWarehouseFactory> stores;
3434
private Specs specs;
3535

3636
@Inject
37-
public WarehouseStoreTransform(List<WarehouseStore> stores, Specs specs) {
37+
public WarehouseStoreTransform(List<FeatureWarehouseFactory> stores, Specs specs) {
3838
this.stores = stores;
3939
this.specs = specs;
4040
}
@@ -48,8 +48,10 @@ public PFeatureRows expand(PFeatureRows input) {
4848
stores,
4949
(featureSpec) -> featureSpec.getDataStores().getWarehouse().getId(),
5050
specs));
51-
output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored")));
52-
output.getErrors().apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("warehouse_errors")));
51+
output.getMain()
52+
.apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored")));
53+
output.getErrors()
54+
.apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("warehouse_errors")));
5355
return output;
5456
}
5557
}

‎ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@
3131
import feast.specs.ImportSpecProto.Field;
3232
import feast.specs.ImportSpecProto.ImportSpec;
3333
import feast.specs.StorageSpecProto.StorageSpec;
34-
import feast.storage.ServingStore;
35-
import feast.storage.WarehouseStore;
36-
import feast.storage.service.ServingStoreService;
37-
import feast.storage.service.WarehouseStoreService;
34+
import feast.store.serving.FeatureServingFactory;
35+
import feast.store.serving.FeatureServingFactoryService;
36+
import feast.store.warehouse.FeatureWarehouseFactory;
37+
import feast.store.warehouse.FeatureWarehouseFactoryService;
3838
import feast.types.FeatureProto.Feature;
3939
import feast.types.FeatureRowProto.FeatureRow;
4040
import feast.types.GranularityProto.Granularity.Enum;
@@ -66,10 +66,10 @@ public void setup() {
6666
featureIds.add(field.getFeatureId());
6767
}
6868
}
69-
for (ServingStore store : ServingStoreService.getAll()) {
69+
for (FeatureServingFactory store : FeatureServingFactoryService.getAll()) {
7070
supportedServingTypes.add(store.getType());
7171
}
72-
for (WarehouseStore store : WarehouseStoreService.getAll()) {
72+
for (FeatureWarehouseFactory store : FeatureWarehouseFactoryService.getAll()) {
7373
supportedWarehouseTypes.add(store.getType());
7474
}
7575
}

‎ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import com.google.cloud.bigquery.StandardSQLTypeName;
2424
import com.google.common.collect.Maps;
2525
import com.google.protobuf.Timestamp;
26-
import feast.storage.bigquery.ValueBigQueryBuilder;
26+
import feast.store.warehouse.bigquery.ValueBigQueryBuilder;
2727
import java.util.Map;
2828
import org.apache.avro.generic.GenericRecord;
2929
import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
@@ -34,8 +34,6 @@
3434
import feast.types.FeatureProto.Feature;
3535
import feast.types.FeatureRowProto.FeatureRow;
3636
import feast.types.ValueProto.Value;
37-
import org.slf4j.Logger;
38-
import org.slf4j.LoggerFactory;
3937

4038
/**
4139
* This is a serializable function used with the BigQueryIO for fetching feature rows directly from

0 commit comments

Comments
 (0)