Skip to content

fix: Pack and unpack repeated list values into and out of arrow array… #263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
piket merged 3 commits into from
Jun 26, 2025
Merged

fix: Pack and unpack repeated list values into and out of arrow array… #263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5c05867
fix: Pack and unpack repeated list values into and out of arrow array…
piket Jun 25, 2025
e3dc7da
Throw error when requested feature service includes both view types
piket Jun 25, 2025
853a3ac
Re-use CopyProtoValuesToArrowArray instead of duplicating switch logic.
piket Jun 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,11 +441,11 @@ test-go: compile-protos-go compile-protos-python install-feast-ci-locally
CGO_ENABLED=1 go test -tags=unit -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html

test-go-integration: compile-protos-go compile-protos-python install-feast-ci-locally
docker compose -f go/integration_tests/valkey/docker-compose.yaml up -d
docker compose -f go/integration_tests/scylladb/docker-compose.yaml up -d
docker compose -f go/internal/feast/integration_tests/valkey/docker-compose.yaml up -d
docker compose -f go/internal/feast/integration_tests/scylladb/docker-compose.yaml up -d
go test -p 1 -tags=integration ./go/internal/...
docker compose -f go/integration_tests/valkey/docker-compose.yaml down
docker compose -f go/integration_tests/scylladb/docker-compose.yaml down
docker compose -f go/internal/feast/integration_tests/valkey/docker-compose.yaml down
docker compose -f go/internal/feast/integration_tests/scylladb/docker-compose.yaml down

format-go:
gofmt -s -w go/
Expand Down
38 changes: 32 additions & 6 deletions go/internal/feast/featurestore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,20 +186,32 @@ func (fs *FeatureStore) GetOnlineFeatures(
fullFeatureNames bool) ([]*onlineserving.FeatureVector, error) {
var err error
var requestedFeatureViews []*onlineserving.FeatureViewAndRefs
var requestedSortedFeatureViews []*onlineserving.SortedFeatureViewAndRefs
var requestedOnDemandFeatureViews []*model.OnDemandFeatureView

// TODO: currently ignores SortedFeatureViews, need to either implement get for them or throw some kind of error/warning
if featureService != nil {
requestedFeatureViews, _, requestedOnDemandFeatureViews, err =
requestedFeatureViews, requestedSortedFeatureViews, requestedOnDemandFeatureViews, err =
onlineserving.GetFeatureViewsToUseByService(featureService, fs.registry, fs.config.Project)
} else {
requestedFeatureViews, _, requestedOnDemandFeatureViews, err =
requestedFeatureViews, requestedSortedFeatureViews, requestedOnDemandFeatureViews, err =
onlineserving.GetFeatureViewsToUseByFeatureRefs(featureRefs, fs.registry, fs.config.Project)
}
if err != nil {
return nil, err
}

if len(requestedSortedFeatureViews) > 0 {
sfvNames := make([]string, len(requestedSortedFeatureViews))
for i, sfv := range requestedSortedFeatureViews {
sfvNames[i] = sfv.View.Base.Name
}
return nil, fmt.Errorf("GetOnlineFeatures does not support sorted feature views %v", sfvNames)
}

if len(requestedFeatureViews) == 0 {
return nil, fmt.Errorf("no feature views found for the requested features")
}

entityColumnMap := make(map[string]*model.Field)
for _, featuresAndView := range requestedFeatureViews {
for _, entityColumn := range featuresAndView.View.EntityColumns {
Expand Down Expand Up @@ -313,17 +325,26 @@ func (fs *FeatureStore) GetOnlineFeaturesRange(

var err error
var requestedSortedFeatureViews []*onlineserving.SortedFeatureViewAndRefs
var requestedFeatureViews []*onlineserving.FeatureViewAndRefs
if featureService != nil {
_, requestedSortedFeatureViews, _, err =
requestedFeatureViews, requestedSortedFeatureViews, _, err =
onlineserving.GetFeatureViewsToUseByService(featureService, fs.registry, fs.config.Project)
} else {
_, requestedSortedFeatureViews, _, err =
requestedFeatureViews, requestedSortedFeatureViews, _, err =
onlineserving.GetFeatureViewsToUseByFeatureRefs(featureRefs, fs.registry, fs.config.Project)
}
if err != nil {
return nil, err
}

if len(requestedFeatureViews) > 0 {
fvNames := make([]string, len(requestedFeatureViews))
for i, fv := range requestedFeatureViews {
fvNames[i] = fv.View.Base.Name
}
return nil, fmt.Errorf("GetOnlineFeaturesRange does not support standard feature views %v", fvNames)
}

if len(requestedSortedFeatureViews) == 0 {
return nil, fmt.Errorf("no sorted feature views found for the requested features")
}
Expand Down Expand Up @@ -453,7 +474,12 @@ func (fs *FeatureStore) ParseFeatures(kind interface{}) (*Features, error) {
}
return &Features{FeaturesRefs: nil, FeatureService: featureService}, nil
case *serving.GetOnlineFeaturesRangeRequest_FeatureService:
return nil, errors.New("range requests only support 'kind' of a list of Features")
featureServiceRequest := kind.(*serving.GetOnlineFeaturesRangeRequest_FeatureService)
featureService, err := fs.registry.GetFeatureService(fs.config.Project, featureServiceRequest.FeatureService)
if err != nil {
return nil, err
}
return &Features{FeaturesRefs: nil, FeatureService: featureService}, nil
default:
return nil, errors.New("cannot parse 'kind' of either a Feature Service or list of Features from request")
}
Expand Down
0 go/integration_tests/scylladb/README.md → ...east/integration_tests/scylladb/README.md
View file
Open in desktop
File renamed without changes.
0 ...ration_tests/scylladb/docker-compose.yaml → ...ration_tests/scylladb/docker-compose.yaml
View file
Open in desktop
File renamed without changes.
0 ...n_tests/scylladb/feature_repo/__init__.py → ...n_tests/scylladb/feature_repo/__init__.py
View file
Open in desktop
File renamed without changes.
0 ..._tests/scylladb/feature_repo/data.parquet → ..._tests/scylladb/feature_repo/data.parquet
View file
Open in desktop
File renamed without changes.
55 changes: 53 additions & 2 deletions ...sts/scylladb/feature_repo/example_repo.py → ...sts/scylladb/feature_repo/example_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from datetime import timedelta

from feast import Entity, FeatureView, Field, FileSource, Project, SortedFeatureView
from feast import Entity, FeatureService, Field, FileSource, Project, SortedFeatureView, FeatureView
from feast.sort_key import SortKey
from feast.protos.feast.core.SortedFeatureView_pb2 import SortOrder
from feast.types import (
Expand Down Expand Up @@ -40,7 +40,53 @@
path="data.parquet", timestamp_field="event_timestamp"
)

mlpfs_test_all_datatypes_view: SortedFeatureView = SortedFeatureView(
mlpfs_test_all_datatypes_view: FeatureView = FeatureView(
name="all_dtypes",
entities=[index_entity],
ttl=timedelta(days=0),
source=mlpfs_test_all_datatypes_source,
tags=tags,
description="Feature View with all supported feast datatypes",
owner=owner,
online=True,
schema=[
Field(name="index_id", dtype=Int64),
Field(name="int_val", dtype=Int32),
Field(name="long_val", dtype=Int64),
Field(name="float_val", dtype=Float32),
Field(name="double_val", dtype=Float64),
Field(name="byte_val", dtype=Bytes),
Field(name="string_val", dtype=String),
Field(name="timestamp_val", dtype=UnixTimestamp),
Field(name="boolean_val", dtype=Bool),
Field(name="array_int_val", dtype=Array(Int32)),
Field(name="array_long_val", dtype=Array(Int64)),
Field(name="array_float_val", dtype=Array(Float32)),
Field(name="array_double_val", dtype=Array(Float64)),
Field(name="array_byte_val", dtype=Array(Bytes)),
Field(name="array_string_val", dtype=Array(String)),
Field(name="array_timestamp_val", dtype=Array(UnixTimestamp)),
Field(name="array_boolean_val", dtype=Array(Bool)),
Field(name="null_int_val", dtype=Int32),
Field(name="null_long_val", dtype=Int64),
Field(name="null_float_val", dtype=Float32),
Field(name="null_double_val", dtype=Float64),
Field(name="null_byte_val", dtype=Bytes),
Field(name="null_string_val", dtype=String),
Field(name="null_timestamp_val", dtype=UnixTimestamp),
Field(name="null_boolean_val", dtype=Bool),
Field(name="null_array_int_val", dtype=Array(Int32)),
Field(name="null_array_long_val", dtype=Array(Int64)),
Field(name="null_array_float_val", dtype=Array(Float32)),
Field(name="null_array_double_val", dtype=Array(Float64)),
Field(name="null_array_byte_val", dtype=Array(Bytes)),
Field(name="null_array_string_val", dtype=Array(String)),
Field(name="null_array_timestamp_val", dtype=Array(UnixTimestamp)),
Field(name="null_array_boolean_val", dtype=Array(Bool)),
],
)

mlpfs_test_all_datatypes_sorted_view: SortedFeatureView = SortedFeatureView(
name="all_dtypes_sorted",
entities=[index_entity],
ttl=timedelta(),
Expand Down Expand Up @@ -93,3 +139,8 @@
Field(name="event_timestamp", dtype=UnixTimestamp),
],
)

mlpfs_test_all_datatypes_service = FeatureService(
name="test_service",
features=[mlpfs_test_all_datatypes_view, mlpfs_test_all_datatypes_sorted_view],
)
0 .../scylladb/feature_repo/feature_store.yaml → .../scylladb/feature_repo/feature_store.yaml
View file
Open in desktop
File renamed without changes.
127 changes: 97 additions & 30 deletions ...rpc_server_read_range_integration_test.go → ...sts/scylladb/scylladb_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
//go:build integration

package server
package scylladb

import (
"context"
"fmt"
"github.com/feast-dev/feast/go/internal/feast/server"
"github.com/feast-dev/feast/go/internal/test"
"github.com/feast-dev/feast/go/protos/feast/serving"
"github.com/feast-dev/feast/go/protos/feast/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"os"
"path/filepath"
"strings"
"testing"
)
Expand All @@ -18,8 +21,12 @@ var client serving.ServingServiceClient
var ctx context.Context

func TestMain(m *testing.M) {
dir := "../../../integration_tests/scylladb/"
err := test.SetupInitializedRepo(dir)
dir, err := filepath.Abs("./")
if err != nil {
fmt.Printf("Failed to get absolute path: %v\n", err)
os.Exit(1)
}
err = test.SetupInitializedRepo(dir)
if err != nil {
fmt.Printf("Failed to set up test environment: %v\n", err)
os.Exit(1)
Expand All @@ -28,7 +35,7 @@ func TestMain(m *testing.M) {
ctx = context.Background()
var closer func()

client, closer = getClient(ctx, "", dir, "")
client, closer = server.GetClient(ctx, "", dir, "")

// Run the tests
exitCode := m.Run()
Expand Down Expand Up @@ -88,30 +95,7 @@ func TestGetOnlineFeaturesRange(t *testing.T) {
}
response, err := client.GetOnlineFeaturesRange(ctx, request)
assert.NoError(t, err)
assert.NotNil(t, response)
assert.Equal(t, 33, len(response.Results))

for i, featureResult := range response.Results {
assert.Equal(t, 3, len(featureResult.Values))
for _, value := range featureResult.Values {
if i == 0 {
// The first result is the entity key which should only have 1 entry
assert.NotNil(t, value)
assert.Equal(t, 1, len(value.Val), "Entity Key should have 1 value, got %d", len(value.Val))
} else {
featureName := featureNames[i-1] // The first entry is the entity key
if strings.Contains(featureName, "null") {
// For null features, we expect the value to contain 1 entry with a nil value
assert.NotNil(t, value)
assert.Equal(t, 1, len(value.Val), "Feature %s should have one values, got %d", featureName, len(value.Val))
assert.Nil(t, value.Val[0].Val, "Feature %s should have a nil value", featureName)
} else {
assert.NotNil(t, value)
assert.Equal(t, 10, len(value.Val), "Feature %s should have 10 values, got %d", featureName, len(value.Val))
}
}
}
}
assertResponseData(t, response, featureNames)
}

func TestGetOnlineFeaturesRange_withEmptySortKeyFilter(t *testing.T) {
Expand Down Expand Up @@ -149,9 +133,92 @@ func TestGetOnlineFeaturesRange_withEmptySortKeyFilter(t *testing.T) {
}
response, err := client.GetOnlineFeaturesRange(ctx, request)
assert.NoError(t, err)
assert.NotNil(t, response)
assert.Equal(t, 33, len(response.Results))
assertResponseData(t, response, featureNames)
}

func TestGetOnlineFeaturesRange_withFeatureService(t *testing.T) {
entities := make(map[string]*types.RepeatedValue)

entities["index_id"] = &types.RepeatedValue{
Val: []*types.Value{
{Val: &types.Value_Int64Val{Int64Val: 1}},
{Val: &types.Value_Int64Val{Int64Val: 2}},
{Val: &types.Value_Int64Val{Int64Val: 3}},
},
}

request := &serving.GetOnlineFeaturesRangeRequest{
Kind: &serving.GetOnlineFeaturesRangeRequest_FeatureService{
FeatureService: "test_service",
},
Entities: entities,
SortKeyFilters: []*serving.SortKeyFilter{
{
SortKeyName: "event_timestamp",
Query: &serving.SortKeyFilter_Range{
Range: &serving.SortKeyFilter_RangeQuery{
RangeStart: &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: 0}},
},
},
},
},
Limit: 10,
}
_, err := client.GetOnlineFeaturesRange(ctx, request)
require.Error(t, err, "Expected an error due to regular feature view requested for range query")
assert.Equal(t, "rpc error: code = Unknown desc = GetOnlineFeaturesRange does not support standard feature views [all_dtypes]", err.Error(), "Expected error message for unsupported feature view")
}

func TestGetOnlineFeaturesRange_withFeatureViewThrowsError(t *testing.T) {
entities := make(map[string]*types.RepeatedValue)

entities["index_id"] = &types.RepeatedValue{
Val: []*types.Value{
{Val: &types.Value_Int64Val{Int64Val: 1}},
{Val: &types.Value_Int64Val{Int64Val: 2}},
{Val: &types.Value_Int64Val{Int64Val: 3}},
},
}

featureNames := []string{"int_val", "long_val", "float_val", "double_val", "byte_val", "string_val", "timestamp_val", "boolean_val",
"null_int_val", "null_long_val", "null_float_val", "null_double_val", "null_byte_val", "null_string_val", "null_timestamp_val", "null_boolean_val",
"null_array_int_val", "null_array_long_val", "null_array_float_val", "null_array_double_val", "null_array_byte_val", "null_array_string_val",
"null_array_boolean_val", "array_int_val", "array_long_val", "array_float_val", "array_double_val", "array_string_val", "array_boolean_val",
"array_byte_val", "array_timestamp_val", "null_array_timestamp_val"}

var featureNamesWithFeatureView []string

for _, featureName := range featureNames {
featureNamesWithFeatureView = append(featureNamesWithFeatureView, "all_dtypes:"+featureName)
}

request := &serving.GetOnlineFeaturesRangeRequest{
Kind: &serving.GetOnlineFeaturesRangeRequest_Features{
Features: &serving.FeatureList{
Val: featureNamesWithFeatureView,
},
},
Entities: entities,
SortKeyFilters: []*serving.SortKeyFilter{
{
SortKeyName: "event_timestamp",
Query: &serving.SortKeyFilter_Range{
Range: &serving.SortKeyFilter_RangeQuery{
RangeStart: &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: 0}},
},
},
},
},
Limit: 10,
}
_, err := client.GetOnlineFeaturesRange(ctx, request)
require.Error(t, err, "Expected an error due to regular feature view requested for range query")
assert.Equal(t, "rpc error: code = Unknown desc = GetOnlineFeaturesRange does not support standard feature views [all_dtypes]", err.Error(), "Expected error message for unsupported feature view")
}

func assertResponseData(t *testing.T, response *serving.GetOnlineFeaturesRangeResponse, featureNames []string) {
assert.NotNil(t, response)
assert.Equal(t, len(featureNames)+1, len(response.Results), "Expected %d results, got %d", len(featureNames)+1, len(response.Results))
for i, featureResult := range response.Results {
assert.Equal(t, 3, len(featureResult.Values))
for _, value := range featureResult.Values {
Expand Down
0 go/integration_tests/valkey/README.md → .../feast/integration_tests/valkey/README.md
View file
Open in desktop
File renamed without changes.
0 ...egration_tests/valkey/docker-compose.yaml → ...egration_tests/valkey/docker-compose.yaml
View file
Open in desktop
File renamed without changes.
0 ...ion_tests/valkey/feature_repo/__init__.py → ...ion_tests/valkey/feature_repo/__init__.py
View file
Open in desktop
File renamed without changes.
0 ...on_tests/valkey/feature_repo/data.parquet → ...on_tests/valkey/feature_repo/data.parquet
View file
Open in desktop
File renamed without changes.
8 changes: 7 additions & 1 deletion ...tests/valkey/feature_repo/example_repo.py → ...tests/valkey/feature_repo/example_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from datetime import timedelta

from feast import Entity, FeatureView, Field, FileSource, Project
from feast import Entity, FeatureView, Field, FileSource, Project, FeatureService
from feast.types import (
Array,
Bool,
Expand Down Expand Up @@ -83,3 +83,9 @@
Field(name="null_array_boolean_val", dtype=Array(Bool)),
],
)

mlpfs_test_all_datatypes_service = FeatureService(
name="test_service",
features=[mlpfs_test_all_datatypes_view],
)

0 ...ts/valkey/feature_repo/feature_store.yaml → ...ts/valkey/feature_repo/feature_store.yaml
View file
Open in desktop
File renamed without changes.
Loading
Loading