Skip to content

feat: Added support for setting env vars in feast services in feast controller #4739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lokeshrangineni merged 1 commit into from
Nov 5, 2024
Merged

feat: Added support for setting env vars in feast services in feast controller #4739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions infra/feast-operator/api/v1alpha1/featurestore_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ type DefaultConfigs struct {

// OptionalConfigs k8s container settings that are optional
type OptionalConfigs struct {
Env *[]corev1.EnvVar `json:"env,omitempty"`
ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
}
Expand Down
11 changes: 11 additions & 0 deletions infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

685 changes: 685 additions & 0 deletions infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml
View file
Open in desktop

Large diffs are not rendered by default.

685 changes: 685 additions & 0 deletions infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml
View file
Open in desktop

Large diffs are not rendered by default.

685 changes: 685 additions & 0 deletions infra/feast-operator/dist/install.yaml
View file
Open in desktop

Large diffs are not rendered by default.

166 changes: 141 additions & 25 deletions infra/feast-operator/internal/controller/featurestore_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package controller
import (
"context"
"encoding/base64"
"reflect"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
Expand Down Expand Up @@ -382,7 +383,9 @@ var _ = Describe("FeatureStore Controller", func() {
Context("When reconciling a resource with all services enabled", func() {
const resourceName = "services"
image := "test:latest"
var pullPolicy corev1.PullPolicy = corev1.PullAlways
var pullPolicy = corev1.PullAlways
var testEnvVarName = "testEnvVarName"
var testEnvVarValue = "testEnvVarValue"

ctx := context.Background()

Expand All @@ -396,29 +399,8 @@ var _ = Describe("FeatureStore Controller", func() {
By("creating the custom resource for the Kind FeatureStore")
err := k8sClient.Get(ctx, typeNamespacedName, featurestore)
if err != nil && errors.IsNotFound(err) {
resource := &feastdevv1alpha1.FeatureStore{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
Spec: feastdevv1alpha1.FeatureStoreSpec{
FeastProject: feastProject,
Services: &feastdevv1alpha1.FeatureStoreServices{
OfflineStore: &feastdevv1alpha1.OfflineStore{},
OnlineStore: &feastdevv1alpha1.OnlineStore{
ServiceConfigs: feastdevv1alpha1.ServiceConfigs{
DefaultConfigs: feastdevv1alpha1.DefaultConfigs{
Image: &image,
},
OptionalConfigs: feastdevv1alpha1.OptionalConfigs{
ImagePullPolicy: &pullPolicy,
Resources: &corev1.ResourceRequirements{},
},
},
},
},
},
}
resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue},
{Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
})
Expand Down Expand Up @@ -463,6 +445,7 @@ var _ = Describe("FeatureStore Controller", func() {
Expect(resource.Status.Applied.Services.OfflineStore.Resources).To(BeNil())
Expect(resource.Status.Applied.Services.OfflineStore.Image).To(Equal(&services.DefaultImage))
Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil())
Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}))
Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy))
Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil())
Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image))
Expand Down Expand Up @@ -658,7 +641,7 @@ var _ = Describe("FeatureStore Controller", func() {
deploy)
Expect(err).NotTo(HaveOccurred())
Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1))
Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1))
Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3))
Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways))
env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env)
Expect(env).NotTo(BeNil())
Expand Down Expand Up @@ -689,6 +672,7 @@ var _ = Describe("FeatureStore Controller", func() {
Registry: regRemote,
}
Expect(repoConfigOnline).To(Equal(onlineConfig))
Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3))

// check client config
cm := &corev1.ConfigMap{}
Expand Down Expand Up @@ -751,6 +735,89 @@ var _ = Describe("FeatureStore Controller", func() {
Expect(repoConfig).To(Equal(testConfig))
})

It("should properly set container env variables", func() {
By("Reconciling the created resource")
Copy link
Contributor

@tchughesiv tchughesiv Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we add an env variable test for the following type of envvar?

     env:
        - name: MY_POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace

Copy link
Contributor Author

@tmihalac tmihalac Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

Copy link
Contributor

@dmartinol dmartinol Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we also support envFrom options to read from Secrets or ConfigMap?
@tchughesiv @tmihalac

Copy link
Contributor

@tchughesiv tchughesiv Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dmartinol that's what this added test will confirm support for... EnvVarSource contains fieldRef, as well as configMapKeyRef & secretKeyRef

Copy link
Contributor Author

@tmihalac tmihalac Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is supported, there is just no specific test for that.
I don't see a lot of added value in adding those tests since the only thing we are doing is overriding the original EnvVar object from the container with the ones from the spec if they have the same name

Copy link
Contributor Author

@tmihalac tmihalac Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tchughesiv

@dmartinol that's what this added test will confirm support for... EnvVarSource contains fieldRef, as well as configMapKeyRef & secretKeyRef

I've added a test for fieldRef, do you think we should add for configMapKeyRef & secretKeyRef as well?
see my previous comment

Copy link
Contributor

@tchughesiv tchughesiv Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tmihalac imo, we don't need a test for the others

controllerReconciler := &FeatureStoreReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}

_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())

resource := &feastdevv1alpha1.FeatureStore{}
err = k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())

req, err := labels.NewRequirement(services.NameLabelKey, selection.Equals, []string{resource.Name})
Expect(err).NotTo(HaveOccurred())
labelSelector := labels.NewSelector().Add(*req)
listOpts := &client.ListOptions{Namespace: resource.Namespace, LabelSelector: labelSelector}
deployList := appsv1.DeploymentList{}
err = k8sClient.List(ctx, &deployList, listOpts)
Expect(err).NotTo(HaveOccurred())
Expect(deployList.Items).To(HaveLen(3))

svcList := corev1.ServiceList{}
err = k8sClient.List(ctx, &svcList, listOpts)
Expect(err).NotTo(HaveOccurred())
Expect(svcList.Items).To(HaveLen(3))

cmList := corev1.ConfigMapList{}
err = k8sClient.List(ctx, &cmList, listOpts)
Expect(err).NotTo(HaveOccurred())
Expect(cmList.Items).To(HaveLen(1))

feast := services.FeastServices{
Client: controllerReconciler.Client,
Context: ctx,
Scheme: controllerReconciler.Scheme,
FeatureStore: resource,
}

fsYamlStr := ""
fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType)
Expect(err).NotTo(HaveOccurred())

// check online config
deploy := &appsv1.Deployment{}
err = k8sClient.Get(ctx, types.NamespacedName{
Name: feast.GetFeastServiceName(services.OnlineFeastType),
Namespace: resource.Namespace,
},
deploy)
Expect(err).NotTo(HaveOccurred())
Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1))
Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3))
Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue())
Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways))

// change feast project and reconcile
resourceNew := resource.DeepCopy()
resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}}
err = k8sClient.Update(ctx, resourceNew)
Expect(err).NotTo(HaveOccurred())
_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())

err = k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())
Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue())
err = k8sClient.Get(ctx, types.NamespacedName{
Name: feast.GetFeastServiceName(services.OnlineFeastType),
Namespace: resource.Namespace,
},
deploy)
Expect(err).NotTo(HaveOccurred())

Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3))
Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue())
})

It("Should delete k8s objects owned by the FeatureStore CR", func() {
By("changing which feast services are configured in the CR")
controllerReconciler := &FeatureStoreReconciler{
Expand Down Expand Up @@ -1122,6 +1189,33 @@ var _ = Describe("FeatureStore Controller", func() {
})
})

func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore {
return &feastdevv1alpha1.FeatureStore{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
Spec: feastdevv1alpha1.FeatureStoreSpec{
FeastProject: feastProject,
Services: &feastdevv1alpha1.FeatureStoreServices{
OfflineStore: &feastdevv1alpha1.OfflineStore{},
OnlineStore: &feastdevv1alpha1.OnlineStore{
ServiceConfigs: feastdevv1alpha1.ServiceConfigs{
DefaultConfigs: feastdevv1alpha1.DefaultConfigs{
Image: &image,
},
OptionalConfigs: feastdevv1alpha1.OptionalConfigs{
Env: envVars,
ImagePullPolicy: &pullPolicy,
Resources: &corev1.ResourceRequirements{},
},
},
},
},
},
}
}

func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar {
for _, e := range envs {
if e.Name == services.FeatureStoreYamlEnvVar {
Expand All @@ -1130,3 +1224,25 @@ func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar {
}
return nil
}

func areEnvVarArraysEqual(arr1 []corev1.EnvVar, arr2 []corev1.EnvVar) bool {
if len(arr1) != len(arr2) {
return false
}

// Create a map to count occurrences of EnvVars in the first array.
envMap := make(map[string]corev1.EnvVar)

for _, env := range arr1 {
envMap[env.Name] = env
}

// Check the second array against the map.
for _, env := range arr2 {
if _, exists := envMap[env.Name]; !exists || !reflect.DeepEqual(envMap[env.Name], env) {
return false
}
}

return true
}
Copy link
Contributor

@tchughesiv tchughesiv Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i wonder if reflect.DeepEqual() might be a better approach? instead of this areEnvVarArraysEqual function?

Copy link
Contributor Author

@tmihalac tmihalac Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tchughesiv

i wonder if reflect.DeepEqual() might be a better approach? instead of this areEnvVarArraysEqual function?

It doesn't take into account 2 arrays are the same if they have the same elements but different order:


import (
	"fmt"
	"reflect"
)

func main() {
	arr1 := [3]int{1, 2, 3}
	arr2 := [3]int{1, 2, 3}
	arr3 := [3]int{3, 2, 1}

	fmt.Println(reflect.DeepEqual(arr1, arr2)) // Output: true
	fmt.Println(reflect.DeepEqual(arr1, arr3)) // Output: false

Copy link
Contributor

@tchughesiv tchughesiv Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the main problem i see with the areEnvVarArraysEqual function is its not checking env.ValueFrom. maybe we use reflect.DeepEqual(envVar, envVar) at the EnvVar level somehow... within this custom function?

Copy link
Contributor Author

@tmihalac tmihalac Nov 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now its done updated the function to !reflect.DeepEqual(envMap[env.Name], env)

Copy link
Contributor

@tchughesiv tchughesiv Nov 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe DeepEqual isn't even needed? envMap[env.Name] == env ??

25 changes: 25 additions & 0 deletions infra/feast-operator/internal/controller/services/services.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,10 +389,35 @@ func (feast *FeastServices) deleteOwnedFeastObj(obj client.Object) error {
}

func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs feastdevv1alpha1.OptionalConfigs) {
if optionalConfigs.Env != nil {
container.Env = mergeEnvVarsArrays(container.Env, optionalConfigs.Env)
}
if optionalConfigs.ImagePullPolicy != nil {
container.ImagePullPolicy = *optionalConfigs.ImagePullPolicy
}
if optionalConfigs.Resources != nil {
container.Resources = *optionalConfigs.Resources
}
}

func mergeEnvVarsArrays(envVars1 []corev1.EnvVar, envVars2 *[]corev1.EnvVar) []corev1.EnvVar {
merged := make(map[string]corev1.EnvVar)

// Add all env vars from the first array
for _, envVar := range envVars1 {
merged[envVar.Name] = envVar
}

// Add all env vars from the second array, overriding duplicates
for _, envVar := range *envVars2 {
merged[envVar.Name] = envVar
}

// Convert the map back to an array
result := make([]corev1.EnvVar, 0, len(merged))
for _, envVar := range merged {
result = append(result, envVar)
}

return result
}
Loading