Skip to content

Commit 2c56521

Browse files
franciscojavierarceofranciscojavierarceo
authored
Merge branch 'master' into feat/add-uuid-feature-types
2 parents d19035c + ea7d4fa commit 2c56521

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2038
-124
lines changed

‎.github/workflows/dbt-integration-tests.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: dbt-integration-tests
2+
3+
# Run dbt integration tests on PRs
4+
on:
5+
pull_request:
6+
types:
7+
- opened
8+
- synchronize
9+
- labeled
10+
paths:
11+
- 'sdk/python/feast/dbt/**'
12+
- 'sdk/python/tests/integration/dbt/**'
13+
- 'sdk/python/tests/unit/dbt/**'
14+
- '.github/workflows/dbt-integration-tests.yml'
15+
16+
jobs:
17+
dbt-integration-test:
18+
if:
19+
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
20+
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
21+
github.event.pull_request.base.repo.full_name == 'feast-dev/feast'
22+
runs-on: ubuntu-latest
23+
strategy:
24+
matrix:
25+
python-version: ["3.11", "3.12"]
26+
env:
27+
PYTHON: ${{ matrix.python-version }}
28+
steps:
29+
- uses: actions/checkout@v4
30+
31+
- name: Setup Python
32+
uses: actions/setup-python@v5
33+
with:
34+
python-version: ${{ matrix.python-version }}
35+
architecture: x64
36+
37+
- name: Install the latest version of uv
38+
uses: astral-sh/setup-uv@v5
39+
with:
40+
enable-cache: true
41+
42+
- name: Install dependencies
43+
run: make install-python-dependencies-ci
44+
45+
- name: Install dbt and dbt-duckdb
46+
run: |
47+
uv pip install --system dbt-core dbt-duckdb
48+
49+
- name: Run dbt integration tests
50+
run: make test-python-integration-dbt
51+
52+
- name: Minimize uv cache
53+
run: uv cache prune --ci

‎.github/workflows/linter.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ jobs:
2222
- name: Install dependencies
2323
run: |
2424
make install-python-dependencies-ci
25+
- name: Cache MyPy
26+
uses: actions/cache@v4
27+
with:
28+
path: sdk/python/.mypy_cache
29+
key: mypy-${{ runner.os }}-py${{ env.PYTHON }}-${{ hashFiles('pyproject.toml', 'setup.py', 'sdk/python/pyproject.toml', 'sdk/python/requirements/*.txt') }}
30+
restore-keys: |
31+
mypy-${{ runner.os }}-py${{ env.PYTHON }}-
32+
mypy-${{ runner.os }}-
2533
- name: Lint python
2634
run: make lint-python
2735
- name: Minimize uv cache

‎.github/workflows/registry-rest-api-tests.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,7 @@ jobs:
145145
- name: Setup and Run Registry Rest API tests
146146
run: |
147147
echo "Running Registry REST API tests..."
148-
cd sdk/python
149-
pytest tests/integration/registration/rest_api/test_registry_rest_api.py --integration -s
148+
uv run pytest sdk/python/tests/integration/registration/rest_api/test_registry_rest_api.py --integration -s
150149
151150
- name: Clean up docker images
152151
if: always()

‎.github/workflows/smoke_tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,6 @@ jobs:
2929
- name: Install dependencies
3030
run: make install-python-dependencies-ci
3131
- name: Test Imports
32-
run: python -c "from feast.cli import cli"
32+
run: uv run python -c "from feast.cli import cli"
3333
- name: Minimize uv cache
3434
run: uv cache prune --ci

‎.github/workflows/unit_tests.yml‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,40 @@ jobs:
3333
uses: astral-sh/setup-uv@v5
3434
with:
3535
enable-cache: true
36+
- name: Install system dependencies
37+
run: |
38+
if [ "$RUNNER_OS" = "Linux" ]; then
39+
sudo apt-get update
40+
sudo apt-get install -y make
41+
elif [ "$RUNNER_OS" = "macOS" ]; then
42+
# make is already installed on macOS runners
43+
which make || brew install make
44+
fi
3645
- name: Install dependencies
3746
run: make install-python-dependencies-ci
3847
- name: Test Python
39-
run: make test-python-unit
48+
run: |
49+
# Set up environment for Ray workers to access packages
50+
export PATH="${{ github.workspace }}/.venv/bin:$PATH"
51+
52+
# Dynamically detect site-packages for uv env
53+
SITE_PACKAGES=$(uv run python -c "import site; print(site.getsitepackages()[0])")
54+
55+
# Preserve any existing PYTHONPATH and add repo + site-packages
56+
export PYTHONPATH="${{ github.workspace }}/sdk/python:${PYTHONPATH}:$SITE_PACKAGES"
57+
58+
echo "Using PYTHONPATH: $PYTHONPATH"
59+
echo "Using PATH: $PATH"
60+
61+
# Ray macOS workarounds
62+
if [[ "$RUNNER_OS" == "macOS" ]]; then
63+
echo "=== Applying macOS Ray compatibility workarounds ==="
64+
export RAY_DISABLE_RUNTIME_ENV_HOOK=1
65+
export PYTHONDONTWRITEBYTECODE=1
66+
echo "Applied macOS workarounds for Python ${{ matrix.python-version }}"
67+
fi
68+
69+
make test-python-unit
4070
- name: Minimize uv cache
4171
run: uv cache prune --ci
4272

‎.pre-commit-config.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
1-
default_stages:
2-
- push
1+
default_stages: [commit]
2+
33
repos:
44
- repo: local
55
hooks:
66
- id: format
77
name: Format
8-
stages: [ push ]
8+
stages: [commit]
99
language: system
1010
entry: make format-python
1111
pass_filenames: false
1212
- id: lint
1313
name: Lint
14-
stages: [ push ]
14+
stages: [commit]
1515
language: system
1616
entry: make lint-python
1717
pass_filenames: false
1818
- id: template
1919
name: Build Templates
20-
stages: [ commit ]
20+
stages: [commit]
2121
language: system
2222
entry: make build-templates
23-
pass_filenames: false
23+
pass_filenames: false

‎Makefile‎

Lines changed: 91 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,36 @@ protos: compile-protos-python compile-protos-docs ## Compile protobufs for Pytho
5555
build: protos build-docker ## Build protobufs and Docker images
5656

5757
format-python: ## Format Python code
58-
cd ${ROOT_DIR}/sdk/python; python -m ruff check --fix feast/ tests/
59-
cd ${ROOT_DIR}/sdk/python; python -m ruff format feast/ tests/
58+
uv run ruff check --fix sdk/python/feast/ sdk/python/tests/
59+
uv run ruff format sdk/python/feast/ sdk/python/tests/
6060

6161
lint-python: ## Lint Python code
62-
cd ${ROOT_DIR}/sdk/python; python -m mypy feast
63-
cd ${ROOT_DIR}/sdk/python; python -m ruff check feast/ tests/
64-
cd ${ROOT_DIR}/sdk/python; python -m ruff format --check feast/ tests
65-
62+
uv run ruff check sdk/python/feast/ sdk/python/tests/
63+
uv run ruff format --check sdk/python/feast/ sdk/python/tests/
64+
uv run bash -c "cd sdk/python && mypy feast"
65+
66+
# New combined target
67+
precommit-check: format-python lint-python ## Run all precommit checks
68+
@echo "✅ All precommit checks passed"
69+
70+
# Install precommit hooks with correct stages
71+
install-precommit: ## Install precommit hooks (runs on commit, not push)
72+
pip install pre-commit
73+
pre-commit install --hook-type pre-commit
74+
@echo "✅ Precommit hooks installed (will run on commit, not push)"
75+
76+
# Manual full type check
77+
mypy-full: ## Full MyPy type checking with all files
78+
uv run bash -c "cd sdk/python && mypy feast tests"
79+
80+
# Run precommit on all files
81+
precommit-all: ## Run all precommit hooks on all files
82+
pre-commit run --all-files
83+
84+
# Make scripts executable
85+
setup-scripts: ## Make helper scripts executable
86+
chmod +x scripts/uv-run.sh scripts/check-init-py.sh scripts/mypy-daemon.sh
87+
6688
##@ Python SDK - local
6789
# formerly install-python-ci-dependencies-uv-venv
6890
# editable install
@@ -74,23 +96,22 @@ install-python-dependencies-minimal: ## Install minimal Python dependencies usin
7496
uv pip sync --require-hashes sdk/python/requirements/py$(PYTHON_VERSION)-minimal-requirements.txt
7597
uv pip install --no-deps -e .[minimal]
7698

77-
##@ Python SDK - system
78-
# the --system flag installs dependencies in the global python context
79-
# instead of a venv which is useful when working in a docker container or ci.
99+
##@ Python SDK - CI (uses uv with virtualenv)
100+
# Uses uv pip sync with virtualenv for CI environments
80101

81102
# Used in github actions/ci
82-
# formerly install-python-ci-dependencies-uv
83-
install-python-dependencies-ci: ## Install Python CI dependencies in system environment using uv
84-
# Install CPU-only torch first to prevent CUDA dependency issues
85-
pip uninstall torch torchvision -y || true
103+
install-python-dependencies-ci: ## Install Python CI dependencies using uv pip sync
104+
# Create virtualenv if it doesn't exist
105+
uv venv .venv
106+
# Install CPU-only torch first to prevent CUDA dependency issues (Linux only)
86107
@if [ "$$(uname -s)" = "Linux" ]; then \
87108
echo "Installing dependencies with torch CPU index for Linux..."; \
88-
uv pip sync --system --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
109+
uv pip sync --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
89110
else \
90111
echo "Installing dependencies from PyPI for macOS..."; \
91-
uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
112+
uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \
92113
fi
93-
uv pip install --system --no-deps -e .
114+
uv pip install --no-deps -e .
94115

95116
# Used in github actions/ci
96117
install-hadoop-dependencies-ci: ## Install Hadoop dependencies
@@ -151,22 +172,44 @@ benchmark-python-local: ## Run integration + benchmark tests for Python (local d
151172
##@ Tests
152173

153174
test-python-unit: ## Run Python unit tests (use pattern=<pattern> to filter tests, e.g., pattern=milvus, pattern=test_online_retrieval.py, pattern=test_online_retrieval.py::test_get_online_features_milvus)
154-
python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") sdk/python/tests
175+
uv run python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") sdk/python/tests
176+
177+
# Fast unit tests only
178+
test-python-unit-fast: ## Run fast unit tests only (no external dependencies)
179+
uv run python -m pytest sdk/python/tests/unit -n auto -x --tb=short
180+
181+
# Changed files only (requires pytest-testmon)
182+
test-python-changed: ## Run tests for changed files only
183+
uv run python -m pytest --testmon -n 8 --tb=short sdk/python/tests
184+
185+
# Quick smoke test for PRs
186+
test-python-smoke: ## Quick smoke test for development
187+
uv run python -m pytest \
188+
sdk/python/tests/unit/test_unit_feature_store.py \
189+
sdk/python/tests/unit/test_repo_operations_validate_feast_project_name.py \
190+
-n 4 --tb=short
155191

156192
test-python-integration: ## Run Python integration tests (CI)
157-
python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
193+
uv run python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
158194
-k "(not snowflake or not test_historical_features_main)" \
159195
-m "not rbac_remote_integration_test" \
160196
--log-cli-level=INFO -s \
161197
sdk/python/tests
162198

199+
# Integration tests with better parallelization
200+
test-python-integration-parallel: ## Run integration tests with enhanced parallelization
201+
uv run python -m pytest sdk/python/tests/integration \
202+
-n auto --dist loadscope \
203+
--timeout=300 --tb=short -v \
204+
--integration --color=yes --durations=20
205+
163206
test-python-integration-local: ## Run Python integration tests (local dev mode)
164207
FEAST_IS_LOCAL_TEST=True \
165208
FEAST_LOCAL_ONLINE_CONTAINER=True \
166209
HADOOP_HOME=$$HOME/hadoop \
167210
CLASSPATH="$$( $$HADOOP_HOME/bin/hadoop classpath --glob ):$$CLASSPATH" \
168211
HADOOP_USER_NAME=root \
169-
python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
212+
uv run python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
170213
-k "not test_lambda_materialization and not test_snowflake_materialization" \
171214
-m "not rbac_remote_integration_test" \
172215
--log-cli-level=INFO -s \
@@ -175,7 +218,7 @@ test-python-integration-local: ## Run Python integration tests (local dev mode)
175218
test-python-integration-rbac-remote: ## Run Python remote RBAC integration tests
176219
FEAST_IS_LOCAL_TEST=True \
177220
FEAST_LOCAL_ONLINE_CONTAINER=True \
178-
python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
221+
uv run python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \
179222
-k "not test_lambda_materialization and not test_snowflake_materialization" \
180223
-m "rbac_remote_integration_test" \
181224
--log-cli-level=INFO -s \
@@ -184,9 +227,31 @@ test-python-integration-rbac-remote: ## Run Python remote RBAC integration tests
184227
test-python-integration-container: ## Run Python integration tests using Docker
185228
@(docker info > /dev/null 2>&1 && \
186229
FEAST_LOCAL_ONLINE_CONTAINER=True \
187-
python -m pytest -n 8 --integration sdk/python/tests \
230+
uv run python -m pytest -n 8 --integration sdk/python/tests \
188231
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
189232

233+
test-python-integration-dbt: ## Run dbt integration tests
234+
@echo "Running dbt integration tests..."
235+
@cd sdk/python/tests/integration/dbt/test_dbt_project && \
236+
echo "Installing dbt dependencies..." && \
237+
dbt deps && \
238+
echo "Building dbt models..." && \
239+
dbt build
240+
@cd sdk/python/tests/integration/dbt && \
241+
echo "Setting up Feast project..." && \
242+
mkdir -p feast_repo/data && \
243+
echo "project: feast_dbt_test\nregistry: data/registry.db\nprovider: local\nonline_store:\n type: sqlite\n path: data/online_store.db" > feast_repo/feature_store.yaml
244+
@cd sdk/python/tests/integration/dbt/feast_repo && \
245+
echo "Testing feast dbt import..." && \
246+
feast dbt import -m ../test_dbt_project/target/manifest.json -e driver_id -d file --tag feast && \
247+
echo "Verifying Feast objects..." && \
248+
feast feature-views list && \
249+
feast entities list
250+
@cd sdk/python && \
251+
echo "Running pytest integration tests..." && \
252+
python -m pytest tests/integration/dbt/test_dbt_integration.py -v --tb=short
253+
@echo "✓ dbt integration tests completed successfully!"
254+
190255
test-python-universal-spark: ## Run Python Spark integration tests
191256
PYTHONPATH='.' \
192257
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \
@@ -220,7 +285,7 @@ test-python-historical-retrieval:
220285
test_historical_features_persisting or \
221286
test_historical_retrieval_fails_on_validation" \
222287
sdk/python/tests
223-
288+
224289
test-python-universal-trino: ## Run Python Trino integration tests
225290
PYTHONPATH='.' \
226291
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \
@@ -556,7 +621,7 @@ test-python-universal-couchbase-online: ## Run Python Couchbase online store int
556621
sdk/python/tests
557622

558623
test-python-universal: ## Run all Python integration tests
559-
python -m pytest -n 8 --integration sdk/python/tests
624+
uv run python -m pytest -n 8 --integration sdk/python/tests
560625

561626
##@ Java
562627

@@ -622,7 +687,7 @@ build-feature-transformation-server-docker: ## Build Feature Transformation Serv
622687
push-feature-server-java-docker: ## Push Feature Server Java Docker image
623688
docker push $(REGISTRY)/feature-server-java:$(VERSION)
624689

625-
build-feature-server-java-docker: ## Build Feature Server Java Docker image
690+
build-feature-server-java-docker: ## Build Feature Server Java Docker image
626691
docker buildx build --build-arg VERSION=$(VERSION) \
627692
-t $(REGISTRY)/feature-server-java:$(VERSION) \
628693
-f java/infra/docker/feature-server/Dockerfile --load .
@@ -727,12 +792,12 @@ build-ui-local: ## Build Feast UI locally
727792
cd $(ROOT_DIR)/ui && yarn install && npm run build --omit=dev
728793
rm -rf $(ROOT_DIR)/sdk/python/feast/ui/build
729794
cp -r $(ROOT_DIR)/ui/build $(ROOT_DIR)/sdk/python/feast/ui/
730-
795+
731796
format-ui: ## Format Feast UI
732797
cd $(ROOT_DIR)/ui && NPM_TOKEN= yarn install && NPM_TOKEN= yarn format
733798

734799

735-
##@ Go SDK
800+
##@ Go SDK
736801
PB_REL = https://github.com/protocolbuffers/protobuf/releases
737802
PB_VERSION = 30.2
738803
PB_ARCH := $(shell uname -m)
@@ -798,4 +863,3 @@ build-go-docker-dev: ## Build Go Docker image for development
798863
docker buildx build --build-arg VERSION=dev \
799864
-t feastdev/feature-server-go:dev \
800865
-f go/infra/docker/feature-server/Dockerfile --load .
801-

‎infra/feast-operator/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,10 @@ test-upgrade:
131131
test-previous-version:
132132
go test -timeout 60m ./test/previous-version/ -v -ginkgo.v
133133

134-
# Requires python3
134+
# Requires python3 with feast installed (use uv run in CI)
135135
.PHONY: test-datasources
136136
test-datasources:
137-
python3 test/data-source-types/data-source-types.py
137+
cd ../.. && uv run python infra/feast-operator/test/data-source-types/data-source-types.py
138138
go test ./test/data-source-types/
139139

140140
.PHONY: lint

0 commit comments

Comments
 (0)