feast-dev · feast-ci-bot · Apr 27, 2022 · Apr 26, 2022 · Apr 27, 2022
diff --git a/.gitignore b/.gitignore
@@ -105,7 +105,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 infra/scripts/*.conf
-go/cmd/server/logging/feature_repo/data/
+go/internal/test/feature_repo/data/
 
 # Translations
 *.mo

@@ -3,20 +3,23 @@ package embedded
 import (
 	"context"
 	"fmt"
-	"github.com/feast-dev/feast/go/internal/feast/server"
-	"github.com/feast-dev/feast/go/internal/feast/server/logging"
-	"github.com/feast-dev/feast/go/protos/feast/serving"
-	"google.golang.org/grpc"
 	"log"
 	"net"
 	"os"
 	"os/signal"
 	"syscall"
 
+	"google.golang.org/grpc"
+
+	"github.com/feast-dev/feast/go/internal/feast/server"
+	"github.com/feast-dev/feast/go/internal/feast/server/logging"
+	"github.com/feast-dev/feast/go/protos/feast/serving"
+
 	"github.com/apache/arrow/go/v8/arrow"
 	"github.com/apache/arrow/go/v8/arrow/array"
 	"github.com/apache/arrow/go/v8/arrow/cdata"
 	"github.com/apache/arrow/go/v8/arrow/memory"
+
 	"github.com/feast-dev/feast/go/internal/feast"
 	"github.com/feast-dev/feast/go/internal/feast/model"
 	"github.com/feast-dev/feast/go/internal/feast/onlineserving"

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"github.com/apache/arrow/go/v8/arrow/memory"
+
 	"github.com/feast-dev/feast/go/internal/feast/model"
 	"github.com/feast-dev/feast/go/internal/feast/onlineserving"
 	"github.com/feast-dev/feast/go/internal/feast/onlinestore"
@@ -287,3 +288,32 @@ func (fs *FeatureStore) readFromOnlineStore(ctx context.Context, entityRows []*p
 	}
 	return fs.onlineStore.OnlineRead(ctx, entityRowsValue, requestedFeatureViewNames, requestedFeatureNames)
 }
+
+func (fs *FeatureStore) GetFcosMap() (map[string]*model.Entity, map[string]*model.FeatureView, map[string]*model.OnDemandFeatureView, error) {
+	odfvs, err := fs.ListOnDemandFeatureViews()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	fvs, err := fs.ListFeatureViews()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	entities, err := fs.ListEntities(true)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	entityMap := make(map[string]*model.Entity)
+	for _, entity := range entities {
+		entityMap[entity.Name] = entity
+	}
+	fvMap := make(map[string]*model.FeatureView)
+	for _, fv := range fvs {
+		fvMap[fv.Base.Name] = fv
+	}
+	odfvMap := make(map[string]*model.OnDemandFeatureView)
+	for _, odfv := range odfvs {
+		odfvMap[odfv.Base.Name] = odfv
+	}
+	return entityMap, fvMap, odfvMap, nil
+}
@@ -6,10 +6,11 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/feast-dev/feast/go/internal/feast/onlinestore"
 	"github.com/feast-dev/feast/go/internal/feast/registry"
 	"github.com/feast-dev/feast/go/protos/feast/types"
-	"github.com/stretchr/testify/assert"
 )
 
 // Return absolute path to the test_repo registry regardless of the working directory

@@ -1,8 +1,9 @@
 package model
 
 import (
-	"github.com/feast-dev/feast/go/protos/feast/core"
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/feast-dev/feast/go/protos/feast/core"
 )
 
 type FeatureService struct {
@@ -11,17 +12,29 @@ type FeatureService struct {
 	CreatedTimestamp     *timestamppb.Timestamp
 	LastUpdatedTimestamp *timestamppb.Timestamp
 	Projections          []*FeatureViewProjection
+	LoggingConfig        *FeatureServiceLoggingConfig
+}
+
+type FeatureServiceLoggingConfig struct {
+	SampleRate float32
 }
 
 func NewFeatureServiceFromProto(proto *core.FeatureService) *FeatureService {
 	projections := make([]*FeatureViewProjection, len(proto.Spec.Features))
 	for index, projectionProto := range proto.Spec.Features {
 		projections[index] = NewFeatureViewProjectionFromProto(projectionProto)
 	}
+	var loggingConfig *FeatureServiceLoggingConfig
+	if proto.GetSpec().GetLoggingConfig() != nil {
+		loggingConfig = &FeatureServiceLoggingConfig{
+			SampleRate: proto.GetSpec().GetLoggingConfig().SampleRate,
+		}
+	}
 	return &FeatureService{Name: proto.Spec.Name,
 		Project:              proto.Spec.Project,
 		CreatedTimestamp:     proto.Meta.CreatedTimestamp,
 		LastUpdatedTimestamp: proto.Meta.LastUpdatedTimestamp,
 		Projections:          projections,
+		LoggingConfig:        loggingConfig,
 	}
 }
@@ -1,9 +1,10 @@
 package model
 
 import (
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+
 	"github.com/feast-dev/feast/go/protos/feast/core"
 	"github.com/feast-dev/feast/go/protos/feast/types"
-	durationpb "google.golang.org/protobuf/types/known/durationpb"
 )
 
 const (

@@ -49,8 +49,10 @@ func (fs *OnDemandFeatureView) NewWithProjection(projection *FeatureViewProjecti
 }
 
 func NewOnDemandFeatureViewFromBase(base *BaseFeatureView) *OnDemandFeatureView {
-
-	featureView := &OnDemandFeatureView{Base: base}
+	featureView := &OnDemandFeatureView{
+		Base:                         base,
+		SourceFeatureViewProjections: map[string]*FeatureViewProjection{},
+		SourceRequestDataSources:     map[string]*core.DataSource_RequestDataOptions{}}
 	return featureView
 }
 

@@ -9,14 +9,15 @@ import (
 
 	"github.com/apache/arrow/go/v8/arrow"
 	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
 	"github.com/feast-dev/feast/go/internal/feast/model"
 	"github.com/feast-dev/feast/go/internal/feast/onlinestore"
 	"github.com/feast-dev/feast/go/protos/feast/serving"
 	prototypes "github.com/feast-dev/feast/go/protos/feast/types"
 	"github.com/feast-dev/feast/go/types"
-	"github.com/golang/protobuf/proto"
-	"google.golang.org/protobuf/types/known/durationpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 /*

@@ -3,12 +3,13 @@ package onlineserving
 import (
 	"testing"
 
-	"github.com/feast-dev/feast/go/internal/feast/model"
-	"github.com/feast-dev/feast/go/protos/feast/core"
-	"github.com/feast-dev/feast/go/protos/feast/types"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/feast-dev/feast/go/internal/feast/model"
+	"github.com/feast-dev/feast/go/protos/feast/core"
+	"github.com/feast-dev/feast/go/protos/feast/types"
 )
 
 func TestGroupingFeatureRefs(t *testing.T) {

@@ -3,11 +3,13 @@ package onlinestore
 import (
 	"context"
 	"fmt"
+
 	"github.com/feast-dev/feast/go/internal/feast/registry"
 
+	"github.com/golang/protobuf/ptypes/timestamp"
+
 	"github.com/feast-dev/feast/go/protos/feast/serving"
 	"github.com/feast-dev/feast/go/protos/feast/types"
-	"github.com/golang/protobuf/ptypes/timestamp"
 )
 
 type FeatureData struct {

@@ -9,12 +9,13 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/feast-dev/feast/go/protos/feast/serving"
-	"github.com/feast-dev/feast/go/protos/feast/types"
 	"github.com/go-redis/redis/v8"
 	"github.com/golang/protobuf/proto"
 	"github.com/spaolacci/murmur3"
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/feast-dev/feast/go/protos/feast/serving"
+	"github.com/feast-dev/feast/go/protos/feast/types"
 )
 
 type redisType int

@@ -5,19 +5,21 @@ import (
 	"database/sql"
 	"encoding/hex"
 	"errors"
-	"github.com/feast-dev/feast/go/internal/feast/registry"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/feast-dev/feast/go/internal/feast/registry"
+
 	"context"
 	"fmt"
 
-	"github.com/feast-dev/feast/go/protos/feast/serving"
-	"github.com/feast-dev/feast/go/protos/feast/types"
 	_ "github.com/mattn/go-sqlite3"
 	"google.golang.org/protobuf/proto"
 	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/feast-dev/feast/go/protos/feast/serving"
+	"github.com/feast-dev/feast/go/protos/feast/types"
 )
 
 type SqliteOnlineStore struct {

@@ -8,17 +8,18 @@ import (
 
 	"github.com/feast-dev/feast/go/internal/feast/registry"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/feast-dev/feast/go/internal/test"
 	"github.com/feast-dev/feast/go/protos/feast/types"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestSqliteAndFeatureRepoSetup(t *testing.T) {
-	dir := "../../test"
+	dir := t.TempDir()
 	feature_repo_path := filepath.Join(dir, "feature_repo")
 	err := test.SetupCleanFeatureRepo(dir)
 	assert.Nil(t, err)
-	defer test.CleanUpRepo(dir)
+
 	config, err := registry.NewRepoConfigFromFile(feature_repo_path)
 	assert.Nil(t, err)
 	assert.Equal(t, "feature_repo", config.Project)
@@ -33,10 +34,10 @@ func TestSqliteAndFeatureRepoSetup(t *testing.T) {
 }
 
 func TestSqliteOnlineRead(t *testing.T) {
-	dir := "../../test"
+	dir := t.TempDir()
 	feature_repo_path := filepath.Join(dir, "feature_repo")
 	test.SetupCleanFeatureRepo(dir)
-	defer test.CleanUpRepo(dir)
+
 	config, err := registry.NewRepoConfigFromFile(feature_repo_path)
 	assert.Nil(t, err)
 	store, err := NewSqliteOnlineStore("feature_repo", config, config.OnlineStore)

@@ -1,13 +1,15 @@
 package registry
 
 import (
-	"github.com/feast-dev/feast/go/protos/feast/core"
-	"github.com/golang/protobuf/proto"
-	"github.com/google/uuid"
-	"google.golang.org/protobuf/types/known/timestamppb"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/google/uuid"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/feast-dev/feast/go/protos/feast/core"
 )
 
 // A LocalRegistryStore is a file-based implementation of the RegistryStore interface.

@@ -2,9 +2,10 @@ package registry
 
 import (
 	"encoding/json"
-	"github.com/ghodss/yaml"
 	"io/ioutil"
 	"path/filepath"
+
+	"github.com/ghodss/yaml"
 )
 
 const (

@@ -1,10 +1,11 @@
 package registry
 
 import (
-	"github.com/stretchr/testify/assert"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestNewRepoConfig(t *testing.T) {

@@ -2,13 +2,15 @@ package server
 
 import (
 	"context"
+	"fmt"
+
+	"github.com/google/uuid"
 
 	"github.com/feast-dev/feast/go/internal/feast"
 	"github.com/feast-dev/feast/go/internal/feast/server/logging"
 	"github.com/feast-dev/feast/go/protos/feast/serving"
 	prototypes "github.com/feast-dev/feast/go/protos/feast/types"
 	"github.com/feast-dev/feast/go/types"
-	"github.com/google/uuid"
 )
 
 const feastServerVersion = "0.0.1"
@@ -55,7 +57,7 @@ func (s *grpcServingServiceServer) GetOnlineFeatures(ctx context.Context, reques
 			FeatureNames: &serving.FeatureList{Val: make([]string, 0)},
 		},
 	}
-	// Entities are currently part of the features as a value and the order that we add it to the resp MetaData
+	// JoinKeys are currently part of the features as a value and the order that we add it to the resp MetaData
 	// Need to figure out a way to map the correct entities to the correct ordering
 	entityValuesMap := make(map[string][]*prototypes.Value, 0)
 	featureNames := make([]string, len(featureVectors))
@@ -76,8 +78,17 @@ func (s *grpcServingServiceServer) GetOnlineFeatures(ctx context.Context, reques
 			EventTimestamps: vector.Timestamps,
 		})
 	}
+
 	if featuresOrService.FeatureService != nil && s.loggingService != nil {
-		go s.loggingService.GenerateLogs(featuresOrService.FeatureService, entityValuesMap, resp.Results[len(request.Entities):], request.RequestContext, requestId)
+		logger, err := s.loggingService.GetOrCreateLogger(featuresOrService.FeatureService)
+		if err != nil {
+			fmt.Printf("Couldn't instantiate logger for feature service %s: %+v", featuresOrService.FeatureService.Name, err)
+		}
+
+		err = logger.Log(entityValuesMap, resp.Results[len(request.Entities):], resp.Metadata.FeatureNames.Val[len(request.Entities):], request.RequestContext, requestId)
+		if err != nil {
+			fmt.Printf("LoggerImpl error[%s]: %+v", featuresOrService.FeatureService.Name, err)
+		}
 	}
 	return resp, nil
 }