feast-dev · felixwang9817 · Oct 21, 2021 · Oct 18, 2021 · Oct 19, 2021 · Oct 19, 2021
diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "deps/feast"]
 	path = deps/feast
 	url = https://github.com/feast-dev/feast
-	branch = v0.9-branch
+	branch = master
diff --git a/common/src/main/java/feast/common/models/FeatureV2.java b/common/src/main/java/feast/common/models/FeatureV2.java
@@ -34,4 +34,17 @@ public static String getFeatureStringRef(FeatureReferenceV2 featureReference) {
     }
     return ref;
   }
+
+  /**
+   * Accepts either a feature reference of the form "featuretable_name:feature_name" or just a
+   * feature name, and returns just the feature name. For example, given either
+   * "driver_hourly_stats:conv_rate" or "conv_rate", "conv_rate" would be returned.
+   *
+   * @param featureReference {String}
+   * @return Base feature name of the feature reference
+   */
+  public static String getFeatureName(String featureReference) {
+    String[] tokens = featureReference.split(":", 2);
+    return tokens[tokens.length - 1];
+  }
 }
diff --git a/deps/feast b/deps/feast
diff --git a/pom.xml b/pom.xml
@@ -269,7 +269,38 @@
                 <version>${grpc.version}</version>
                 <scope>test</scope>
             </dependency>
-
+
+            <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-java-root -->
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-java-root</artifactId>
+                <version>5.0.0</version>
+                <type>pom</type>
+            </dependency>
+
+            <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-vector -->
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-vector</artifactId>
+                <version>5.0.0</version>
+            </dependency>
+
+            <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory -->
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-memory</artifactId>
+                <version>5.0.0</version>
+                <type>pom</type>
+            </dependency>
+
+            <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory-netty -->
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-memory-netty</artifactId>
+                <version>5.0.0</version>
+                <scope>runtime</scope>
+            </dependency>
+
             <!-- HTTP/REST -->
             <dependency>
                 <groupId>io.swagger</groupId>

diff --git a/serving/pom.xml b/serving/pom.xml
@@ -269,6 +269,37 @@
       <version>1.10.2</version>
     </dependency>
 
+    <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-java-root -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-java-root</artifactId>
+      <version>5.0.0</version>
+      <type>pom</type>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-vector -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-vector</artifactId>
+      <version>5.0.0</version>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory</artifactId>
+      <version>5.0.0</version>
+      <type>pom</type>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory-netty -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-netty</artifactId>
+      <version>5.0.0</version>
+      <scope>runtime</scope>
+    </dependency>
+
     <!-- Utilities -->
     <dependency>
       <groupId>com.fasterxml.jackson.dataformat</groupId>

diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java
@@ -82,6 +82,16 @@ public void setRegistry(final String registry) {
     this.registry = registry;
   }
 
+  private String transformationServiceEndpoint;
+
+  public String getTransformationServiceEndpoint() {
+    return transformationServiceEndpoint;
+  }
+
+  public void setTransformationServiceEndpoint(final String transformationServiceEndpoint) {
+    this.transformationServiceEndpoint = transformationServiceEndpoint;
+  }
+
   private CoreAuthenticationProperties coreAuthentication;
 
   public CoreAuthenticationProperties getCoreAuthentication() {

diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java
@@ -23,6 +23,7 @@
 import com.google.protobuf.AbstractMessageLite;
 import feast.serving.registry.LocalRegistryRepo;
 import feast.serving.service.OnlineServingServiceV2;
+import feast.serving.service.OnlineTransformationService;
 import feast.serving.service.ServingServiceV2;
 import feast.serving.specs.CachedSpecService;
 import feast.serving.specs.CoreFeatureSpecRetriever;
@@ -126,7 +127,13 @@ public ServingServiceV2 servingServiceV2(
     log.info("Created CoreFeatureSpecRetriever");
     featureSpecRetriever = new CoreFeatureSpecRetriever(specService);
 
-    servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever);
+    final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint();
+    final OnlineTransformationService onlineTransformationService =
+        new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever);
+
+    servingService =
+        new OnlineServingServiceV2(
+            retrieverV2, tracer, featureSpecRetriever, onlineTransformationService);
 
     return servingService;
   }
@@ -164,7 +171,13 @@ public ServingServiceV2 registryBasedServingServiceV2(
     final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry()));
     featureSpecRetriever = new RegistryFeatureSpecRetriever(repo);
 
-    servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever);
+    final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint();
+    final OnlineTransformationService onlineTransformationService =
+        new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever);
+
+    servingService =
+        new OnlineServingServiceV2(
+            retrieverV2, tracer, featureSpecRetriever, onlineTransformationService);
 
     return servingService;
   }

diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java
@@ -18,14 +18,22 @@
 
 import feast.proto.core.FeatureProto;
 import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
 import feast.proto.core.RegistryProto;
 import feast.proto.serving.ServingAPIProto;
 import feast.serving.exception.SpecRetrievalException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class LocalRegistryRepo implements RegistryRepository {
   private final RegistryProto.Registry registry;
+  private Map<String, FeatureViewProto.FeatureViewSpec> featureViewNameToSpec;
+  private Map<String, OnDemandFeatureViewProto.OnDemandFeatureViewSpec>
+      onDemandFeatureViewNameToSpec;
 
   public LocalRegistryRepo(Path localRegistryPath) {
     if (!localRegistryPath.toFile().exists()) {
@@ -38,6 +46,26 @@ public LocalRegistryRepo(Path localRegistryPath) {
     } catch (final Exception e) {
       throw new RuntimeException(e);
     }
+
+    final RegistryProto.Registry registry = this.getRegistry();
+    List<FeatureViewProto.FeatureViewSpec> featureViewSpecs =
+        registry.getFeatureViewsList().stream()
+            .map(fv -> fv.getSpec())
+            .collect(Collectors.toList());
+    featureViewNameToSpec =
+        featureViewSpecs.stream()
+            .collect(
+                Collectors.toMap(FeatureViewProto.FeatureViewSpec::getName, Function.identity()));
+    List<OnDemandFeatureViewProto.OnDemandFeatureViewSpec> onDemandFeatureViewSpecs =
+        registry.getOnDemandFeatureViewsList().stream()
+            .map(odfv -> odfv.getSpec())
+            .collect(Collectors.toList());
+    onDemandFeatureViewNameToSpec =
+        onDemandFeatureViewSpecs.stream()
+            .collect(
+                Collectors.toMap(
+                    OnDemandFeatureViewProto.OnDemandFeatureViewSpec::getName,
+                    Function.identity()));
   }
 
   @Override
@@ -48,15 +76,12 @@ public RegistryProto.Registry getRegistry() {
   @Override
   public FeatureViewProto.FeatureViewSpec getFeatureViewSpec(
       String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
-    final RegistryProto.Registry registry = this.getRegistry();
-    for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) {
-      if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) {
-        return featureView.getSpec();
-      }
+    String featureViewName = featureReference.getFeatureTable();
+    if (featureViewNameToSpec.containsKey(featureViewName)) {
+      return featureViewNameToSpec.get(featureViewName);
     }
     throw new SpecRetrievalException(
-        String.format(
-            "Unable to find feature view with name: %s", featureReference.getFeatureTable()));
+        String.format("Unable to find feature view with name: %s", featureViewName));
   }
 
   @Override
@@ -75,4 +100,22 @@ public FeatureProto.FeatureSpecV2 getFeatureSpec(
             "Unable to find feature with name: %s in feature view: %s",
             featureReference.getName(), featureReference.getFeatureTable()));
   }
+
+  @Override
+  public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
+      String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
+    String onDemandFeatureViewName = featureReference.getFeatureTable();
+    if (onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName)) {
+      return onDemandFeatureViewNameToSpec.get(onDemandFeatureViewName);
+    }
+    throw new SpecRetrievalException(
+        String.format(
+            "Unable to find on demand feature view with name: %s", onDemandFeatureViewName));
+  }
+
+  @Override
+  public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) {
+    String onDemandFeatureViewName = featureReference.getFeatureTable();
+    return onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName);
+  }
 }
diff --git a/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/serving/src/main/java/feast/serving/registry/RegistryRepository.java
@@ -18,6 +18,7 @@
 
 import feast.proto.core.FeatureProto;
 import feast.proto.core.FeatureViewProto;
+import feast.proto.core.OnDemandFeatureViewProto;
 import feast.proto.core.RegistryProto;
 import feast.proto.serving.ServingAPIProto;
 
@@ -33,4 +34,9 @@ FeatureViewProto.FeatureViewSpec getFeatureViewSpec(
 
   FeatureProto.FeatureSpecV2 getFeatureSpec(
       String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+  OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
+      String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
+
+  boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference);
 }