feast-dev · zhilingc · Feb 8, 2019 · Feb 8, 2019 · Feb 8, 2019 · Feb 9, 2019
diff --git a/.gitignore b/.gitignore
@@ -46,4 +46,122 @@ vendor
 .direnv
 
 .terraform/
-*.tfvars
+*.tfvars
+
+# python 
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+.vscode
diff --git a/Makefile b/Makefile
@@ -1,9 +1,28 @@
-VERSION_FILE=VERSION
-FEAST_VERSION=`cat $(VERSION_FILE)`
+# 
+#  Copyright 2019 The Feast Authors
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+# 
+#      https://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# 
+
+VERSION_FILE                = VERSION
+FEAST_VERSION               = `cat $(VERSION_FILE)`
 
 test:
 	mvn test
 
+test-integration:
+	$(MAKE) -C testing/integration test-integration ID=$(ID)
+
 build-deps:
 	$(MAKE) -C protos gen-go
 	dep ensure

diff --git a/core/src/test/java/feast/core/CoreApplicationTest.java b/core/src/test/java/feast/core/CoreApplicationTest.java
@@ -48,7 +48,7 @@
  */
 @RunWith(SpringRunner.class)
 @SpringBootTest(properties = {
-    "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}",
+    "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}",
     "spring.datasource.url=jdbc:h2:mem:testdb",
     "feast.store.warehouse.type=file.json",
     "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}",

diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoServingTest.java
@@ -49,7 +49,7 @@
  */
 @RunWith(SpringRunner.class)
 @SpringBootTest(properties = {
-    "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}",
+    "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}",
     "spring.datasource.url=jdbc:h2:mem:testdb",
     "feast.store.warehouse.type=file.json",
     "feast.store.warehouse.options={\"path\":\"/tmp/foobar\"}",

diff --git a/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java b/core/src/test/java/feast/core/CoreApplicationWithNoWarehouseTest.java
@@ -49,7 +49,7 @@
  */
 @RunWith(SpringRunner.class)
 @SpringBootTest(properties = {
-    "feast.jobs.workspace=${java.io.tmpdir}${random.uuid}",
+    "feast.jobs.workspace=${java.io.tmpdir}/${random.uuid}",
     "spring.datasource.url=jdbc:h2:mem:testdb",
     "feast.store.serving.type=redis",
     "feast.store.serving.options={\"host\":\"localhost\",\"port\":1234}",

diff --git a/testing/Makefile b/testing/Makefile
@@ -32,6 +32,6 @@ update-config: get-cluster-credentials
 update-plugins: get-cluster-credentials
 	kubectl create configmap plugins --from-file=plugins.yaml=prow/plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f -
 
-build-push:
-	docker build test-image/ -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION)
+build-push-test-image:
+	docker build -f docker-images/test-image/Dockerfile -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION) .
 	$(PUSH) "$(REGISTRY)/$(PROJECT)/test-image:$(VERSION)"
diff --git a/testing/README.md b/testing/README.md
@@ -4,9 +4,9 @@ This folder contains the Feast test infrastructure.
 
 ## Components
 
-* test-image/ - Base docker image and script for running tests.
+* docker-images/ - Docker images for running tests
 * prow/ - Prow configuration (plugins and jobs)
-* tf/ - Terraform modules to provision the base testing infrastructure on GCP
+* infrastructure/ - Terraform modules to provision the base testing infrastructure on GCP
 
 ## Set up
 
@@ -23,7 +23,7 @@ mv tf/gcs
 terraform import google_storage_bucket.kf-feast-terraform-state kf-feast-terraform-state
 ```
 
-4. Ensure that all variables are set correctly in `tf/terraform.tfvars`. It is likely that the GCP project will need to be updated.
+4. Ensure that all variables are set correctly in `infrastructure/terraform.tfvars`. It is likely that the GCP project will need to be updated.
 
 5. Create the primary Kubernetes cluster which will host Prow and Argo
 
@@ -51,7 +51,11 @@ To update Prow jobs, plugins, or the Docker image used for testing, modify one o
 
 - `prow/config.yaml`
 - `prow/plugins.yaml`
-- `test-image/Dockerfile`
-- `test-image/run.sh`
+- `docker-images/test-image/Dockerfile`
+- `docker-images/test-image/run.sh`
 
-After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry.
+After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry.
+
+## Installing Argo
+
+Argo is used to run integration tests. It follows the [standard installation process described in the argo quickstart](https://github.com/argoproj/argo/blob/master/demo.md), with the artifact repository [configured to use gcs](https://github.com/argoproj/argo/blob/master/ARTIFACT_REPO.md).
diff --git a/testing/docker-images/docker-builder-image/Dockerfile b/testing/docker-images/docker-builder-image/Dockerfile
@@ -0,0 +1,25 @@
+FROM docker:18.09
+
+# from https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/alpine/Dockerfile
+ENV CLOUD_SDK_VERSION=236.0.0
+
+ENV PATH /google-cloud-sdk/bin:$PATH
+RUN apk --no-cache add \
+        curl \
+        python \
+        py-crcmod \
+        bash \
+        libc6-compat \
+        openssh-client \
+        git \
+        gnupg \
+    && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
+    tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
+    rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
+    ln -s /lib /lib64 && \
+    gcloud config set core/disable_usage_reporting true && \
+    gcloud config set component_manager/disable_update_check true && \
+    gcloud config set metrics/environment github_docker_image && \
+    gcloud --version
+
+RUN gcloud auth configure-docker 
diff --git a/testing/docker-images/terraform-image/Dockerfile b/testing/docker-images/terraform-image/Dockerfile
@@ -0,0 +1,19 @@
+FROM debian:stretch
+
+RUN apt-get update && apt-get install -y jq wget python zip netcat
+
+RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-233.0.0-linux-x86_64.tar.gz | tar xzf - \
+  && /google-cloud-sdk/bin/gcloud -q components install kubectl \
+  && /google-cloud-sdk/bin/gcloud -q components install beta \
+  && /google-cloud-sdk/bin/gcloud -q components install bq
+
+ENV PATH="/google-cloud-sdk/bin:${PATH}"
+
+RUN wget https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip \
+  && unzip terraform_0.11.11_linux_amd64.zip \
+  && mv terraform /bin/terraform
+
+RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-arm64.tar.gz | tar xzf - \
+  && mv linux-arm64/helm /bin/helm
+
+RUN helm init --client-only
diff --git a/testing/test-image/Dockerfile → testing/docker-images/test-image/Dockerfile b/testing/test-image/Dockerfile → testing/docker-images/test-image/Dockerfile
diff --git a/testing/test-image/run.sh → testing/docker-images/test-image/run.sh b/testing/test-image/run.sh → testing/docker-images/test-image/run.sh
diff --git a/testing/tf/cloud-build/cloud-build.tf → ...infrastructure/cloud-build/cloud-build.tf b/testing/tf/cloud-build/cloud-build.tf → ...infrastructure/cloud-build/cloud-build.tf
diff --git a/testing/tf/cloud-build/cloudbuild.yaml → ...nfrastructure/cloud-build/cloudbuild.yaml b/testing/tf/cloud-build/cloudbuild.yaml → ...nfrastructure/cloud-build/cloudbuild.yaml
diff --git a/testing/tf/cloud-build/variables.tf → ...g/infrastructure/cloud-build/variables.tf b/testing/tf/cloud-build/variables.tf → ...g/infrastructure/cloud-build/variables.tf
diff --git a/testing/tf/gcs/backend.tf → testing/infrastructure/gcs/backend.tf b/testing/tf/gcs/backend.tf → testing/infrastructure/gcs/backend.tf
diff --git a/testing/tf/gcs/main.tf → testing/infrastructure/gcs/main.tf b/testing/tf/gcs/main.tf → testing/infrastructure/gcs/main.tf
diff --git a/testing/tf/k8s-cluster/backend.tf → ...ing/infrastructure/k8s-cluster/backend.tf b/testing/tf/k8s-cluster/backend.tf → ...ing/infrastructure/k8s-cluster/backend.tf
diff --git a/testing/tf/k8s-cluster/main.tf → testing/infrastructure/k8s-cluster/main.tf b/testing/tf/k8s-cluster/main.tf → testing/infrastructure/k8s-cluster/main.tf
@@ -16,6 +16,10 @@ resource "google_container_cluster" "primary-test-cluster" {
   node_config {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/devstorage.read_write",
+      "https://www.googleapis.com/auth/devstorage.full_control",
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/bigquery.insertdata",
     ]
 
     machine_type = "n1-standard-4"

diff --git a/testing/tf/k8s-cluster/output.tf → testing/infrastructure/k8s-cluster/output.tf b/testing/tf/k8s-cluster/output.tf → testing/infrastructure/k8s-cluster/output.tf
diff --git a/testing/tf/k8s-cluster/provider.tf → ...ng/infrastructure/k8s-cluster/provider.tf b/testing/tf/k8s-cluster/provider.tf → ...ng/infrastructure/k8s-cluster/provider.tf
diff --git a/testing/tf/k8s-cluster/variables.tf → ...g/infrastructure/k8s-cluster/variables.tf b/testing/tf/k8s-cluster/variables.tf → ...g/infrastructure/k8s-cluster/variables.tf
diff --git a/testing/integration/Makefile b/testing/integration/Makefile
@@ -0,0 +1,23 @@
+# 
+#  Copyright 2019 The Feast Authors
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+# 
+#      https://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# 
+
+.PHONY: test-integration it-redis-bq-dataflow
+
+test-integration:
+	@$(MAKE) it-redis-bq-dataflow ID=$(ID)
+
+it-redis-bq-dataflow:
+	argo submit --watch tests/redis-bq-dataflow/workflow.yaml -p prId=$(ID)
diff --git a/testing/integration/README.md b/testing/integration/README.md
@@ -0,0 +1,29 @@
+# Integration tests
+
+Integration tests for Feast are run on argo workflows. The tests follow the following steps:
+
+1. Provision infrastructure using Terraform
+2. Test using pytest
+3. Teardown infrastructure using Terraform
+
+## Terraform modules
+
+The `tf/modules` directory contains Terraform modules to set up the necessary infrastructure. Currently contains:
+
+- `cluster`: kubernetes cluster with necessary permissions to run Feast jobs
+- `feast-helm`: Feast helm installation.
+
+## Tests
+
+The `tests` directory contains the integration tests. Each folder should contain the following:
+
+1. The terraform scripts to set up the necessary infra.
+2. Data to run the tests on
+3. A pytest file that executes the ingestion jobs and then tests for correctness.
+4. Argo workflow yaml to orchestrate the entire process.
+
+Multiple tests can be run on the same infrastructure.
+
+### Adding new tests
+
+To add your own tests, either (1) create a new test case within an existing folder or (2) create a new folder with the resources mentioned above. 
diff --git a/testing/integration/tests/redis-bq-dataflow/.env.local b/testing/integration/tests/redis-bq-dataflow/.env.local
@@ -0,0 +1,7 @@
+# Example environment variables that need to be set when running this test locally
+# Export the variables like so:
+# set -a; source .env.local
+FEAST_CORE_URL=localhost:6565
+FEAST_SERVING_URL=localhost:6566
+PROJECT_ID=kf-feast
+BUCKET_NAME=it-feast-storage
diff --git a/testing/integration/tests/redis-bq-dataflow/README.md b/testing/integration/tests/redis-bq-dataflow/README.md
@@ -0,0 +1,9 @@
+# redis-bq-dataflow
+
+This repository contains an end-to-end test that sets up the following infrastructure:
+
+1. Feast configured to run jobs on dataflow
+2. Redis as serving store
+3. BQ as warehouse store
+
+And then runs the tests in `test_feast.py`.
diff --git a/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml b/testing/integration/tests/redis-bq-dataflow/data/entity/myentity.yaml
@@ -0,0 +1,5 @@
+name: myentity
+description: my test entity
+tags:
+  - tag1
+  - tag2
diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutedoubleredis1.yaml
@@ -0,0 +1,7 @@
+id: myentity.feature_double_redis
+name: feature_double_redis
+entity: myentity
+owner: bob@example.com
+description: test entity.
+valueType:  DOUBLE
+uri: https://github.com/bob/example
diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minutefloatredis1.yaml
@@ -0,0 +1,7 @@
+id: myentity.feature_float_redis
+name: feature_float_redis
+entity: myentity
+owner: bob@example.com
+description: test entity.
+valueType:  FLOAT
+uri: https://github.com/bob/example
diff --git a/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml b/testing/integration/tests/redis-bq-dataflow/data/feature/feature_minuteint32redis1.yaml
@@ -0,0 +1,7 @@
+id: myentity.feature_int32_redis
+name: feature_int32_redis
+entity: myentity
+owner: bob@example.com
+description: test entity.
+valueType:  INT32
+uri: https://github.com/bob/example