feast-dev · feast-ci-bot · Mar 27, 2020 · Mar 25, 2020 · Mar 25, 2020 · Mar 25, 2020
diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java
@@ -137,7 +137,7 @@ public static PipelineResult runPipeline(ImportOptions options) throws IOExcepti
 
       // Step 3. Write FeatureRow to the corresponding Store.
       WriteResult writeFeatureRows =
-          validatedRows.get(FEATURE_ROW_OUT).apply("WriteFeatureRowToStore", featureSink.write());
+          validatedRows.get(FEATURE_ROW_OUT).apply("WriteFeatureRowToStore", featureSink.writer());
 
       // Step 4. Write FailedElements to a dead letter table in BigQuery.
       if (options.getDeadLetterTableSpec() != null) {

diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java
@@ -162,6 +162,9 @@ public static void publishFeatureRowsToKafka(
    * Create a Feature Row with random value according to the FeatureSetSpec
    *
    * <p>See {@link #createRandomFeatureRow(FeatureSetSpec, int)}
+   *
+   * @param featureSetSpec {@link FeatureSetSpec}
+   * @return {@link FeatureRow}
    */
   public static FeatureRow createRandomFeatureRow(FeatureSetSpec featureSetSpec) {
     ThreadLocalRandom random = ThreadLocalRandom.current();

diff --git a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java
@@ -25,15 +25,11 @@
 import feast.core.StoreProto.Store.RedisConfig;
 import feast.core.StoreProto.Store.Subscription;
 import feast.serving.FeastProperties;
-import feast.serving.service.BatchServingService;
-import feast.serving.service.JobService;
-import feast.serving.service.NoopJobService;
-import feast.serving.service.OnlineServingService;
-import feast.serving.service.ServingService;
+import feast.serving.service.*;
 import feast.serving.specs.CachedSpecService;
-import feast.storage.api.retrieval.BatchRetriever;
+import feast.storage.api.retrieval.HistoricalRetriever;
 import feast.storage.api.retrieval.OnlineRetriever;
-import feast.storage.connectors.bigquery.retrieval.BigQueryBatchRetriever;
+import feast.storage.connectors.bigquery.retrieval.BigQueryHistoricalRetriever;
 import feast.storage.connectors.redis.retrieval.RedisOnlineRetriever;
 import io.opentracing.Tracer;
 import java.util.Map;
@@ -109,8 +105,8 @@ public ServingService servingService(
               "Unable to instantiate jobService for BigQuery store.");
         }
 
-        BatchRetriever bqRetriever =
-            BigQueryBatchRetriever.builder()
+        HistoricalRetriever bqRetriever =
+            BigQueryHistoricalRetriever.builder()
                 .setBigquery(bigquery)
                 .setDatasetId(bqConfig.getDatasetId())
                 .setProjectId(bqConfig.getProjectId())
@@ -121,7 +117,7 @@ public ServingService servingService(
                 .setStorage(storage)
                 .build();
 
-        servingService = new BatchServingService(bqRetriever, specService, jobService);
+        servingService = new HistoricalServingService(bqRetriever, specService, jobService);
         break;
       case CASSANDRA:
       case UNRECOGNIZED:

diff --git a/.../serving/service/BatchServingService.java → ...ing/service/HistoricalServingService.java b/.../serving/service/BatchServingService.java → ...ing/service/HistoricalServingService.java
@@ -18,25 +18,29 @@
 
 import feast.serving.ServingAPIProto;
 import feast.serving.ServingAPIProto.*;
+import feast.serving.ServingAPIProto.Job.Builder;
 import feast.serving.specs.CachedSpecService;
-import feast.storage.api.retrieval.BatchRetriever;
 import feast.storage.api.retrieval.FeatureSetRequest;
-import feast.storage.connectors.bigquery.retrieval.BigQueryBatchRetriever;
+import feast.storage.api.retrieval.HistoricalRetrievalResult;
+import feast.storage.api.retrieval.HistoricalRetriever;
+import feast.storage.connectors.bigquery.retrieval.BigQueryHistoricalRetriever;
 import io.grpc.Status;
 import java.util.List;
 import java.util.Optional;
+import java.util.UUID;
 import org.slf4j.Logger;
 
-public class BatchServingService implements ServingService {
+public class HistoricalServingService implements ServingService {
 
-  private static final Logger log = org.slf4j.LoggerFactory.getLogger(BatchServingService.class);
+  private static final Logger log =
+      org.slf4j.LoggerFactory.getLogger(HistoricalServingService.class);
 
-  private final BatchRetriever retriever;
+  private final HistoricalRetriever retriever;
   private final CachedSpecService specService;
   private final JobService jobService;
 
-  public BatchServingService(
-      BatchRetriever retriever, CachedSpecService specService, JobService jobService) {
+  public HistoricalServingService(
+      HistoricalRetriever retriever, CachedSpecService specService, JobService jobService) {
     this.retriever = retriever;
     this.specService = specService;
     this.jobService = jobService;
@@ -47,10 +51,11 @@ public BatchServingService(
   public GetFeastServingInfoResponse getFeastServingInfo(
       GetFeastServingInfoRequest getFeastServingInfoRequest) {
     try {
-      BigQueryBatchRetriever bigQueryBatchRetriever = (BigQueryBatchRetriever) retriever;
+      BigQueryHistoricalRetriever bigQueryHistoricalRetriever =
+          (BigQueryHistoricalRetriever) retriever;
       return GetFeastServingInfoResponse.newBuilder()
           .setType(FeastServingType.FEAST_SERVING_TYPE_BATCH)
-          .setJobStagingLocation(bigQueryBatchRetriever.jobStagingLocation())
+          .setJobStagingLocation(bigQueryHistoricalRetriever.jobStagingLocation())
           .build();
     } catch (Exception e) {
       return GetFeastServingInfoResponse.newBuilder()
@@ -70,9 +75,28 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getF
   public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) {
     List<FeatureSetRequest> featureSetRequests =
         specService.getFeatureSets(getFeaturesRequest.getFeaturesList());
-    Job feastJob = retriever.getBatchFeatures(getFeaturesRequest, featureSetRequests);
-    jobService.upsert(feastJob);
-    return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build();
+    String retrievalId = UUID.randomUUID().toString();
+    Job runningJob =
+        Job.newBuilder()
+            .setId(retrievalId)
+            .setType(JobType.JOB_TYPE_DOWNLOAD)
+            .setStatus(JobStatus.JOB_STATUS_RUNNING)
+            .build();
+    jobService.upsert(runningJob);
+    Thread thread =
+        new Thread(
+            new Runnable() {
+              @Override
+              public void run() {
+                HistoricalRetrievalResult result =
+                    retriever.getHistoricalFeatures(
+                        retrievalId, getFeaturesRequest.getDatasetSource(), featureSetRequests);
+                jobService.upsert(resultToJob(result));
+              }
+            });
+    thread.start();
+
+    return GetBatchFeaturesResponse.newBuilder().setJob(runningJob).build();
   }
 
   /** {@inheritDoc} */
@@ -86,4 +110,19 @@ public GetJobResponse getJob(GetJobRequest getJobRequest) {
     }
     return GetJobResponse.newBuilder().setJob(job.get()).build();
   }
+
+  private Job resultToJob(HistoricalRetrievalResult result) {
+    Builder builder =
+        Job.newBuilder()
+            .setId(result.getId())
+            .setType(JobType.JOB_TYPE_DOWNLOAD)
+            .setStatus(result.getStatus());
+    if (result.hasError()) {
+      return builder.setError(result.getError()).build();
+    }
+    return builder
+        .addAllFileUris(result.getFileUris())
+        .setDataFormat(result.getDataFormat())
+        .build();
+  }
 }
diff --git a/storage/api/src/main/java/feast/storage/api/retrieval/HistoricalRetrievalResult.java b/storage/api/src/main/java/feast/storage/api/retrieval/HistoricalRetrievalResult.java
@@ -0,0 +1,100 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.retrieval;
+
+import com.google.auto.value.AutoValue;
+import feast.serving.ServingAPIProto.DataFormat;
+import feast.serving.ServingAPIProto.JobStatus;
+import java.io.Serializable;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/** Result of a historical feature retrieval request. */
+@AutoValue
+public abstract class HistoricalRetrievalResult implements Serializable {
+
+  public abstract String getId();
+
+  public abstract JobStatus getStatus();
+
+  @Nullable
+  public abstract String getError();
+
+  @Nullable
+  public abstract List<String> getFileUris();
+
+  @Nullable
+  public abstract DataFormat getDataFormat();
+
+  /**
+   * Instantiates a {@link HistoricalRetrievalResult} indicating that the retrieval was a failure,
+   * together with its associated error.
+   *
+   * @param id retrieval id identifying the retrieval request.
+   * @param error error that occurred
+   * @return {@link HistoricalRetrievalResult}
+   */
+  public static HistoricalRetrievalResult error(String id, Exception error) {
+    return newBuilder()
+        .setId(id)
+        .setStatus(JobStatus.JOB_STATUS_DONE)
+        .setError(error.getMessage())
+        .build();
+  }
+
+  /**
+   * Instantiates a {@link HistoricalRetrievalResult} indicating that the retrieval was a success,
+   * together with the location of the output.
+   *
+   * @param id retrieval id identifying the retrieval request
+   * @param fileUris list of output file URIs
+   * @param dataFormat data format of the output files
+   * @return
+   */
+  public static HistoricalRetrievalResult success(
+      String id, List<String> fileUris, DataFormat dataFormat) {
+    return newBuilder()
+        .setId(id)
+        .setStatus(JobStatus.JOB_STATUS_DONE)
+        .setFileUris(fileUris)
+        .setDataFormat(dataFormat)
+        .build();
+  }
+
+  static Builder newBuilder() {
+    return new AutoValue_HistoricalRetrievalResult.Builder();
+  }
+
+  @AutoValue.Builder
+  abstract static class Builder {
+    abstract Builder setId(String id);
+
+    abstract Builder setStatus(JobStatus jobStatus);
+
+    abstract Builder setError(String error);
+
+    abstract Builder setFileUris(List<String> fileUris);
+
+    abstract Builder setDataFormat(DataFormat dataFormat);
+
+    abstract HistoricalRetrievalResult build();
+  }
+
+  public boolean hasError() {
+    return getError() != null;
+  }
+}
diff --git a/...storage/api/retrieval/BatchRetriever.java → ...ge/api/retrieval/HistoricalRetriever.java b/...storage/api/retrieval/BatchRetriever.java → ...ge/api/retrieval/HistoricalRetriever.java
@@ -16,22 +16,26 @@
  */
 package feast.storage.api.retrieval;
 
-import feast.serving.ServingAPIProto;
+import feast.serving.ServingAPIProto.DatasetSource;
 import java.util.List;
 
-/** Interface for implementing user defined retrieval functionality from Batch/historical stores. */
-public interface BatchRetriever {
+/**
+ * A historical retriever is a feature retriever that retrieves feature data corresponding to
+ * provided entities over a given period of time.
+ */
+public interface HistoricalRetriever {
 
   /**
    * Get all features corresponding to the provided batch features request.
    *
-   * @param request {@link ServingAPIProto.GetBatchFeaturesRequest} containing requested features
-   *     and file containing entity columns.
+   * @param retrievalId String that uniquely identifies this retrieval request.
+   * @param datasetSource {@link DatasetSource} containing source to load the dataset containing
+   *     entity columns.
    * @param featureSetRequests List of {@link FeatureSetRequest} to feature references in the
    *     request tied to that feature set.
-   * @return {@link ServingAPIProto.Job} if successful, contains the location of the results, else
-   *     contains the error to be returned to the user.
+   * @return {@link HistoricalRetrievalResult} if successful, contains the location of the results,
+   *     else contains the error to be returned to the user.
    */
-  ServingAPIProto.Job getBatchFeatures(
-      ServingAPIProto.GetBatchFeaturesRequest request, List<FeatureSetRequest> featureSetRequests);
+  HistoricalRetrievalResult getHistoricalFeatures(
+      String retrievalId, DatasetSource datasetSource, List<FeatureSetRequest> featureSetRequests);
 }
diff --git a/storage/api/src/main/java/feast/storage/api/retrieval/OnlineRetriever.java b/storage/api/src/main/java/feast/storage/api/retrieval/OnlineRetriever.java
@@ -20,7 +20,10 @@
 import feast.types.FeatureRowProto.FeatureRow;
 import java.util.List;
 
-/** Interface for implementing user defined retrieval functionality from Online stores. */
+/**
+ * An online retriever is a feature retriever that retrieves the latest feature data corresponding
+ * to provided entities.
+ */
 public interface OnlineRetriever {
 
   /**
@@ -29,7 +32,8 @@ public interface OnlineRetriever {
    * @param entityRows list of entity rows in the feature request
    * @param featureSetRequests List of {@link FeatureSetRequest} to feature references in the
    *     request tied to that feature set.
-   * @return list of {@link OnlineRetrieverResponse} for each entity row
+   * @return list of lists of {@link FeatureRow}s corresponding to each feature set request and
+   *     entity row.
    */
   List<List<FeatureRow>> getOnlineFeatures(
       List<EntityRow> entityRows, List<FeatureSetRequest> featureSetRequests);

diff --git a/storage/api/src/main/java/feast/storage/api/retrieval/OnlineRetrieverResponse.java b/storage/api/src/main/java/feast/storage/api/retrieval/OnlineRetrieverResponse.java
diff --git a/storage/api/src/main/java/feast/storage/api/write/FeatureSink.java b/storage/api/src/main/java/feast/storage/api/write/FeatureSink.java
@@ -50,5 +50,5 @@ public interface FeatureSink extends Serializable {
    *
    * @return {@link PTransform}
    */
-  PTransform<PCollection<FeatureRow>, WriteResult> write();
+  PTransform<PCollection<FeatureRow>, WriteResult> writer();
 }
diff --git a/storage/api/src/main/java/feast/storage/api/write/WriteResult.java b/storage/api/src/main/java/feast/storage/api/write/WriteResult.java
@@ -34,7 +34,15 @@ public final class WriteResult implements Serializable, POutput {
   private static TupleTag<FeatureRow> successfulInsertsTag = new TupleTag<>("successfulInserts");
   private static TupleTag<FailedElement> failedInsertsTupleTag = new TupleTag<>("failedInserts");
 
-  /** Creates a {@link WriteResult} in the given {@link Pipeline}. */
+  /**
+   * Creates a {@link WriteResult} in the given {@link Pipeline}.
+   *
+   * @param pipeline {@link Pipeline}
+   * @param successfulInserts {@link PCollection} of {@link FeatureRow}s successfully inserted into
+   *     the store
+   * @param failedInserts {@link PCollection} of {@link FailedElement}s
+   * @return {@link WriteResult}
+   */
   public static WriteResult in(
       Pipeline pipeline,
       PCollection<FeatureRow> successfulInserts,

diff --git a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java b/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java
@@ -33,7 +33,8 @@ public class TestUtil {
   /**
    * Create a Feature Row with random value according to the FeatureSetSpec
    *
-   * <p>See {@link #createRandomFeatureRow(FeatureSet, int)}
+   * @param featureSet {@link FeatureSet}
+   * @return {@link FeatureRow}
    */
   public static FeatureRow createRandomFeatureRow(FeatureSet featureSet) {
     ThreadLocalRandom random = ThreadLocalRandom.current();