feast-dev · franciscojavierarceo · Jun 30, 2022 · Jun 30, 2022 · Jun 30, 2022 · Jun 30, 2022
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -1,7 +1,13 @@
 name: build_wheels
 
 # Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml"
-on: [workflow_dispatch, workflow_call]
+# Developers who are starting a new release should use this workflow to ensure wheels will be built correctly.
+# Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly.
+on:
+  workflow_dispatch:
+      tags:
+        - 'v*.*.*'
+  workflow_call:
 
 jobs:
   get-version:
@@ -60,15 +66,20 @@ jobs:
         uses: pypa/cibuildwheel@v2.7.0
         env:
           CIBW_BUILD: "cp3*_x86_64"
-          CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
+          CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64"
           CIBW_ARCHS: "native"
           CIBW_ENVIRONMENT: >
             COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
           CIBW_BEFORE_ALL_LINUX: |
             curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
             tar -C /usr/local -xzf go.tar.gz
             go version
+            yum -y update &&
+            yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm &&
+            yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm &&
+            yum install -y --enablerepo=epel arrow-devel # For C++
           CIBW_BEFORE_ALL_MACOS: |
+            brew install apache-arrow
             curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
             sudo installer -pkg python.pkg -target /
           # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
@@ -93,6 +104,7 @@ jobs:
           CIBW_BEFORE_BUILD: |
             git status
             git restore go.mod go.sum
+            brew install apache-arrow
       - uses: actions/upload-artifact@v2
         with:
           name: wheels
@@ -134,11 +146,11 @@ jobs:
 
   verify-python-wheels:
     runs-on: ${{ matrix.os }}
-    needs: [build-python-wheel, build-source-distribution]
+    needs: [build-python-wheel, build-source-distribution, get-version]
     strategy:
       matrix:
         os: [ubuntu-latest,  macos-10.15 ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10"]
+        python-version: [ "3.8", "3.9", "3.10"]
         from-source: [ True, False ]
     env:
       # this script is for testing servers
@@ -153,6 +165,7 @@ jobs:
         else
            echo "Succeeded!"
         fi
+      VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
     steps:
       - name: Setup Python
         id: setup-python
@@ -174,12 +187,24 @@ jobs:
           cd dist/
           pip install wheel
           for f in *.whl; do pip install $f || true; done
+      - name: Install apache-arrow on ubuntu
+        if: ${{ matrix.from-source && matrix.os == 'ubuntu-latest' }}
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V libarrow-dev
+      - name: Install apache-arrow on macos
+        if: ${{ matrix.from-source && matrix.os == 'macos-10.15' && matrix.python-version != '3.10' }}
+        run: brew install apache-arrow
       - name: Install dist with go
         if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}}
         env:
           COMPILE_GO: "True"
         run: |
-          pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
+          pip install 'grpcio-tools==1.48.0' 'pybindgen==0.22.0'
           go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
           go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
           pip install dist/*tar.gz
@@ -190,6 +215,20 @@ jobs:
       - name: Install OS X dependencies
         if: matrix.os == 'macos-10.15'
         run: brew install coreutils
+      # Validate that the feast version installed is not development and is the correct version of the tag we ran it off of.
+      - name: Validate Feast Version
+        run: |
+          VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+'
+          OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$'
+          VERSION_OUTPUT=$(feast version)
+          VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX")
+          OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX")
+          if  [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then
+            echo "Correct Feast Version Installed"
+          else
+            echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION."
+            exit 1
+          fi
       - name: Smoke test
         run: |
           feast init test_repo
@@ -206,5 +245,5 @@ jobs:
           feast apply
           echo "$TEST_SCRIPT" > run-and-wait.sh
           pip install cffi
-          printf "\ngo_feature_retrieval: True" >> feature_store.yaml
+          printf "\ngo_feature_serving: True" >> feature_store.yaml
           bash run-and-wait.sh feast serve
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -39,6 +39,14 @@ jobs:
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
       - name: Install pip-tools
         run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V libarrow-dev
       - name: Install dependencies
         run: |
           make compile-protos-go
@@ -63,7 +71,13 @@ jobs:
       - name: Upgrade pip version
         run: |
           pip install --upgrade "pip>=21.3.1,<22.1"
-      - name: Install dependencies
-        run:  make install-go-proto-dependencies
+      - name: Install apache-arrow on ubuntu
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V libarrow-dev
       - name: Lint go
         run: make lint-go
diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -127,6 +127,18 @@ jobs:
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
       - name: Install pip-tools
         run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V libarrow-dev
+      - name: Install apache-arrow on macos
+        if: matrix.os == 'macOS-latest'
+        run: brew install apache-arrow
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Setup Redis Cluster

diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
@@ -0,0 +1,191 @@
+name: nightly-ci
+
+on:
+  schedule:
+    - cron: '00 08 * * *'  # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST.
+
+# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532
+#concurrency:
+#  group: pr-integration-tests-${{ github.event.pull_request.number }}
+#  cancel-in-progress: true
+
+jobs:
+  check_date:
+    runs-on: ubuntu-latest
+    name: Check latest commit
+    outputs:
+      WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+      - id: check_date
+        name: Check if there were commits in the last day
+        if: ${{ github.event_name == 'schedule' }}
+        run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
+  build-docker-image:
+    needs: [check_date]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: master
+          submodules: recursive
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+      - name: Set up AWS SDK
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+      - name: Set ECR image tag
+        id: image-tag
+        run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
+      - name: Cache Public ECR Image
+        id: lambda_python_3_9
+        uses: actions/cache@v2
+        with:
+          path: ~/cache
+          key: lambda_python_3_9
+      - name: Handle Cache Miss (pull public ECR image & save it to tar file)
+        if: steps.cache-primes.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ~/cache
+          docker pull public.ecr.aws/lambda/python:3.9
+          docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar
+      - name: Handle Cache Hit (load docker image from tar file)
+        if: steps.cache-primes.outputs.cache-hit == 'true'
+        run: |
+          docker load -i ~/cache/lambda_python_3_9.tar
+      - name: Build and push
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: feast-python-server
+        run: |
+          docker build \
+            --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
+            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
+            --load \
+            .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
+    outputs:
+      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
+  integration-test-python:
+    needs: [check_date, build-docker-image]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.8" ]
+        os: [ ubuntu-latest ]
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+          submodules: recursive
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18.0
+      - name: Set up gcloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+      - name: Use gcloud CLI
+        run: gcloud info
+      - name: Set up AWS SDK
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Use AWS CLI
+        run: aws sts get-caller-identity
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade "pip>=21.3.1,<22.1"
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ steps.pip-cache.outputs.dir }}
+            /opt/hostedtoolcache/Python
+            /Users/runner/hostedtoolcache/Python
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
+          restore-keys: |
+            ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
+      - name: Install apache-arrow on ubuntu
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+            sudo apt update
+            sudo apt install -y -V ca-certificates lsb-release wget
+            wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+            sudo apt update
+            sudo apt install -y -V libarrow-dev
+      - name: Install apache-arrow on macos
+        if: matrix.os == 'macOS-latest'
+        run: brew install apache-arrow
+      - name: Install dependencies
+        run: make install-python-ci-dependencies
+      - name: Setup Redis Cluster
+        run: |
+          docker pull vishnunair/docker-redis-cluster:latest
+          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
+      - name: Test python
+        if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
+        env:
+          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
+          FEAST_USAGE: "False"
+          IS_TEST: "True"
+          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
+          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
+          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
+          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
+        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: integrationtests
+          env_vars: OS,PYTHON
+          fail_ci_if_error: true
+          verbose: true