feast-dev · feast-ci-bot · May 5, 2020 · May 1, 2020 · May 1, 2020 · May 1, 2020
diff --git a/.github/workflows/docker_compose_tests.yml b/.github/workflows/docker_compose_tests.yml
@@ -0,0 +1,12 @@
+name: docker compose tests
+
+on: [push, pull_request]
+
+jobs:
+  basic-redis-e2e-tests-docker-compose:
+    runs-on: ubuntu-latest
+    name: basic redis e2e tests on docker compose
+    steps:
+      - uses: actions/checkout@v2
+      - name: test docker compose
+        run: ./infra/scripts/test-docker-compose.sh
diff --git a/.github/workflows/unit-tests.yml → .github/workflows/unit_tests.yml b/.github/workflows/unit-tests.yml → .github/workflows/unit_tests.yml
@@ -35,16 +35,13 @@
     "import os\n",
     "\n",
     "# Feast Core acts as the central feature registry\n",
-    "FEAST_CORE_URL = os.getenv('FEAST_CORE_URL', 'core:6565')\n",
+    "FEAST_CORE_URL = os.getenv('FEAST_CORE_URL', 'localhost:6565')\n",
     "\n",
     "# Feast Online Serving allows for the retrieval of real-time feature data\n",
-    "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'online-serving:6566')\n",
+    "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'localhost:6566')\n",
     "\n",
     "# Feast Batch Serving allows for the retrieval of historical feature data\n",
-    "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'batch-serving:6567')\n",
-    "\n",
-    "# PYTHON_REPOSITORY_PATH is the path to the Python SDK inside the Feast Git Repo\n",
-    "PYTHON_REPOSITORY_PATH = os.getenv('PYTHON_REPOSITORY_PATH', '../../')"
+    "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'localhost:6567')"
    ]
   },
   {
@@ -67,44 +64,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!python -m pip install --ignore-installed --upgrade feast"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "(Alternative) Install from local repository"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import sys\n",
-    "os.environ['PYTHON_SDK_PATH'] = os.path.join(PYTHON_REPOSITORY_PATH, 'sdk/python')\n",
-    "sys.path.append(os.environ['PYTHON_SDK_PATH'])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!echo $PYTHON_SDK_PATH"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!python -m pip install --ignore-installed --upgrade -e  ${PYTHON_SDK_PATH}"
+    "!pip install feast"
    ]
   },
   {
@@ -501,17 +461,8 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.7.4"
-  },
-  "pycharm": {
-   "stem_cell": {
-    "cell_type": "raw",
-    "metadata": {
-     "collapsed": false
-    },
-    "source": []
-   }
   }
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
@@ -1,21 +1,23 @@
 # General
 COMPOSE_PROJECT_NAME=feast
 FEAST_VERSION=latest
+FEAST_REPOSITORY_VERSION=v0.4.7
 
 # Feast Core
 FEAST_CORE_IMAGE=gcr.io/kf-feast/feast-core
 FEAST_CORE_CONFIG=direct-runner.yml
 FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
 
-# Feast Serving
-FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
-FEAST_ONLINE_SERVING_CONFIG=online-serving.yml
-FEAST_ONLINE_STORE_CONFIG=redis-store.yml
+# Feast Serving Batch (BigQuery)
 FEAST_BATCH_SERVING_CONFIG=batch-serving.yml
 FEAST_BATCH_STORE_CONFIG=bq-store.yml
 FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
-FEAST_JOB_STAGING_LOCATION=gs://your-gcs-bucket/staging
+FEAST_BATCH_JOB_STAGING_LOCATION=gs://your-gcs-bucket/staging
 
-# Jupyter
-FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
+# Feast Serving Online (Redis)
+FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
+FEAST_ONLINE_SERVING_CONFIG=online-serving.yml
+FEAST_ONLINE_STORE_CONFIG=redis-store.yml
 
+# Jupyter
+FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
@@ -0,0 +1,29 @@
+version: "3.7"
+
+services:
+  batch-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./serving/${FEAST_BATCH_SERVING_CONFIG}:/etc/feast/application.yml
+      - ./serving/${FEAST_BATCH_STORE_CONFIG}:/etc/feast/store.yml
+      - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
+    depends_on:
+      - redis
+    ports:
+      - 6567:6567
+    restart: on-failure
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+      FEAST_JOB_STAGING_LOCATION: ${FEAST_BATCH_JOB_STAGING_LOCATION}
+    command:
+      - "java"
+      - "-Xms1024m"
+      - "-Xmx1024m"
+      - "-jar"
+      - "/opt/feast/feast-serving.jar"
+      - "--spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml"
+
+  redis:
+    image: redis:5-alpine
+    ports:
+      - "6379:6379"
@@ -0,0 +1,23 @@
+version: "3.7"
+
+services:
+  online-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./serving/${FEAST_ONLINE_SERVING_CONFIG}:/etc/feast/application.yml
+      - ./serving/${FEAST_ONLINE_STORE_CONFIG}:/etc/feast/store.yml
+    depends_on:
+      - redis
+    ports:
+      - 6566:6566
+    restart: on-failure
+    command:
+      - java
+      - -jar
+      - /opt/feast/feast-serving.jar
+      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
+
+  redis:
+    image: redis:5-alpine
+    ports:
+      - "6379:6379"
@@ -21,68 +21,22 @@ services:
       - /opt/feast/feast-core.jar
       - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
 
-  online-serving:
-    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
-    volumes:
-      - ./serving/${FEAST_ONLINE_SERVING_CONFIG}:/etc/feast/application.yml
-      - ./serving/${FEAST_ONLINE_STORE_CONFIG}:/etc/feast/store.yml
-    depends_on:
-      - core
-      - redis
-    ports:
-      - 6566:6566
-    restart: on-failure
-    command:
-      - java
-      - -jar
-      - /opt/feast/feast-serving.jar
-      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
-
-  batch-serving:
-    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
-    volumes:
-      - ./serving/${FEAST_BATCH_SERVING_CONFIG}:/etc/feast/application.yml
-      - ./serving/${FEAST_BATCH_STORE_CONFIG}:/etc/feast/store.yml
-      - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
-    depends_on:
-      - core
-      - redis
-    ports:
-      - 6567:6567
-    restart: on-failure
-    environment:
-      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
-      FEAST_JOB_STAGING_LOCATION: ${FEAST_JOB_STAGING_LOCATION}
-    command:
-      - "java"
-      - "-Xms1024m"
-      - "-Xmx1024m"
-      - "-jar"
-      - "/opt/feast/feast-serving.jar"
-      - "--spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml"
-
   jupyter:
-    image: jupyter/datascience-notebook:63d0df23b673
+    image: jupyter/minimal-notebook:619e9cc2fc07
     volumes:
-      - ../../:/home/jovyan/feast
       - ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
+      - ./jupyter/startup.sh:/etc/startup.sh
     depends_on:
       - core
-      - online-serving
     environment:
       FEAST_CORE_URL: core:6565
-      FEAST_SERVING_URL: online-serving:6566
+      FEAST_ONLINE_SERVING_URL: online-serving:6566
+      FEAST_BATCH_SERVING_URL: batch-serving:6567
       GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+      FEAST_REPOSITORY_VERSION: ${FEAST_REPOSITORY_VERSION}
     ports:
       - 8888:8888
-    command:
-      - start-notebook.sh
-      - --NotebookApp.token=''
-
-  redis:
-    image: redis:5-alpine
-    ports:
-      - "6379:6379"
+    command: ["/etc/startup.sh"]
 
   kafka:
     image: confluentinc/cp-kafka:5.2.1

@@ -1 +1,4 @@
-{}
+{
+  "type": "service_account",
+  "project_id": "just-some-project"
+}
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Clone Feast repository into Jupyter container
+git clone -b ${FEAST_REPOSITORY_VERSION} --single-branch https://github.com/gojek/feast.git || true
+
+# Install CI requirements (only needed for running tests)
+pip install -r feast/sdk/python/requirements-ci.txt
+
+# Install Feast SDK
+pip install -e feast/sdk/python -U
+
+# Start Jupyter Notebook
+start-notebook.sh --NotebookApp.token=''
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "
+============================================================
+Running Docker Compose tests with pytest at 'tests/e2e'
+============================================================
+"
+
+export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
+export COMPOSE_INTERACTIVE_NO_CLI=1
+
+# Create Docker Compose configuration file
+cd ${PROJECT_ROOT_DIR}/infra/docker-compose/
+cp .env.sample .env
+
+# Start Docker Compose containers
+docker-compose -f docker-compose.yml -f docker-compose.online.yml up -d
+
+# Get Jupyter container IP address
+export JUPYTER_DOCKER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jupyter_1)
+
+# Print Jupyter container information
+docker inspect feast_jupyter_1
+docker logs feast_jupyter_1
+
+# Wait for Jupyter Notebook Container to come online
+${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${JUPYTER_DOCKER_CONTAINER_IP_ADDRESS}:8888 --timeout=300
+
+# Run e2e tests for Redis
+docker exec feast_jupyter_1 bash -c 'cd feast/tests/e2e/ && pytest -s basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online-serving:6566'
+
+# Shut down docker-compose images
+docker-compose -f docker-compose.yml -f docker-compose.online.yml down