Skip to content

ci: Maturin-related builds from source #5270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tchughesiv merged 1 commit into from
Apr 17, 2025
Merged

ci: Maturin-related builds from source #5270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .dockerignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,5 @@ charts
env
**/.terraform
offline_build/
!offline_build/arrow
!offline_build/ibis
26 changes: 12 additions & 14 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,27 +85,25 @@ lock-python-dependencies-all:
# Remove all existing requirements because we noticed the lock file is not always updated correctly.
# Removing and running the command again ensures that the lock file is always up to date.
rm -rf sdk/python/requirements/* 2>/dev/null || true
pixi run --environment $(call get_env_name,3.11) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p 3.11 --no-strip-extras setup.py --extra addtl-sources \
--generate-hashes --output-file sdk/python/requirements/py3.11-addtl-sources-requirements.txt"
pixi run --environment $(call get_env_name,3.11) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p 3.11 --no-strip-extras setup.py --extra pandas-build \
--generate-hashes --output-file sdk/python/requirements/py3.11-pandas-requirements.txt"
pixi run --environment $(call get_env_name,3.11) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p 3.11 --no-strip-extras setup.py --extra minimal-sdist-build \
--no-emit-package milvus-lite \
--no-emit-package psycopg-binary \
--output-file sdk/python/requirements/py3.11-sdist-requirements.txt"
$(foreach ver,$(PYTHON_VERSIONS),\
pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p $(ver) --no-strip-extras setup.py --extra minimal \
--generate-hashes --output-file sdk/python/requirements/py$(ver)-minimal-requirements.txt" && \
pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p $(ver) --no-strip-extras setup.py --extra ci \
--generate-hashes --output-file sdk/python/requirements/py$(ver)-ci-requirements.txt" && \
pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p $(ver) --no-strip-extras setup.py \
--generate-hashes --output-file sdk/python/requirements/py$(ver)-requirements.txt" && \
pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p $(ver) --no-strip-extras setup.py --extra minimal \
--generate-hashes --output-file sdk/python/requirements/py$(ver)-minimal-requirements.txt" && \
pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip compile -p $(ver) --no-strip-extras setup.py --extra minimal-sdist-build \
--no-emit-package milvus-lite \
--generate-hashes --output-file sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \
pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \
"uv pip install -p $(ver) pybuild-deps==0.5.0 && \
pybuild-deps compile --generate-hashes \
-o sdk/python/requirements/py$(ver)-minimal-sdist-requirements-build.txt \
sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \
) true

compile-protos-python:
Expand Down
47 changes: 38 additions & 9 deletions infra/scripts/offline-binary-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,34 @@ cd ${PROJECT_ROOT_DIR}
rm -rf ./offline_build
mkdir offline_build

alias cachi2='docker run --rm -ti -v "$PWD:$PWD:z" -w "$PWD" quay.io/konflux-ci/cachi2:f7a61b067f4446e4982d0e3b9545ce4aa0d8284f'
cachi2 fetch-deps \
--output ${OFFLINE_BUILD_DIR}/cachi2-output \
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
# not needed for downstream build from release
###############################
# yarn builder
docker build \
--tag yarn-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yarn \
sdk/python/feast/infra/feature_servers/multicloud/offline

hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui-output \
'{
"type": "yarn",
"path": "ui"
}'
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui.env --for-output-dir /tmp/hermeto-yarn-ui-output ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui-output

hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-yarn-output \
'{
"type": "yarn",
"path": "sdk/python/feast/ui"
}'
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto-yarn.env --for-output-dir /tmp/hermeto-yarn-output ${OFFLINE_BUILD_DIR}/hermeto-yarn-output
###############################

hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-output \
'{
"type": "pip",
"path": ".",
Expand All @@ -20,20 +45,24 @@ cachi2 fetch-deps \
],
"requirements_build_files": [
"sdk/python/requirements/py3.11-minimal-requirements.txt",
"sdk/python/requirements/py3.11-sdist-requirements.txt"
"sdk/python/requirements/py3.11-minimal-sdist-requirements.txt",
"sdk/python/requirements/py3.11-minimal-sdist-requirements-build.txt"
],
"allow_binary": "true"
}'

cachi2 generate-env ${OFFLINE_BUILD_DIR}/cachi2-output -o ${OFFLINE_BUILD_DIR}/cachi2.env --for-output-dir /tmp/cachi2-output
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto.env --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output

# feast OFFLINE builder
docker build \
--volume ${OFFLINE_BUILD_DIR}/cachi2-output:/tmp/cachi2-output:Z \
--volume ${OFFLINE_BUILD_DIR}/cachi2.env:/tmp/cachi2.env:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui-output:/tmp/hermeto-yarn-ui-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui.env:/tmp/hermeto-yarn-ui.env:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn-output:/tmp/hermeto-yarn-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn.env:/tmp/hermeto-yarn.env:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
--network none \
--tag feature-server:binary-build \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.binary \
--load sdk/python/feast/infra/feature_servers/multicloud
${PROJECT_ROOT_DIR}

docker run --rm -ti feature-server:binary-build feast version
46 changes: 46 additions & 0 deletions infra/scripts/offline-binary-release-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# to run -> source ./infra/scripts/offline-binary-release-build.sh
# on the build host... requires docker/podman

# Get Feast project repository root directory
PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
OFFLINE_BUILD_DIR=${PROJECT_ROOT_DIR}/offline_build
cd ${PROJECT_ROOT_DIR}

rm -rf ./offline_build
mkdir offline_build

# yarn builder
docker build \
--build-arg RELEASE=true \
--tag yarn-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yarn \
sdk/python/feast/infra/feature_servers/multicloud/offline

alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-output \
'{
"type": "pip",
"path": ".",
"requirements_files": [
"sdk/python/feast/infra/feature_servers/multicloud/requirements.txt"
],
"requirements_build_files": [
"sdk/python/requirements/py3.11-minimal-requirements.txt",
"sdk/python/requirements/py3.11-minimal-sdist-requirements.txt",
"sdk/python/requirements/py3.11-minimal-sdist-requirements-build.txt"
],
"allow_binary": "true"
}'
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto.env --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output

# feast OFFLINE builder
docker build \
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
--network none \
--tag feature-server:binary-release-build \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.binary.release \
sdk/python/feast/infra/feature_servers/multicloud

docker run --rm -ti feature-server:binary-release-build feast version
89 changes: 45 additions & 44 deletions infra/scripts/offline-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

APACHE_ARROW_VERSION="17.0.0"
SUBSTRAIT_VERSION="0.44.0"
IBIS_VERSION="9.5.0"

# Get Feast project repository root directory
PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
Expand All @@ -16,81 +17,81 @@ mkdir offline_build
docker build \
--tag yum-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum \
--load sdk/python/feast/infra/feature_servers/multicloud/offline
sdk/python/feast/infra/feature_servers/multicloud/offline

git clone --branch apache-arrow-${APACHE_ARROW_VERSION} https://github.com/apache/arrow ${OFFLINE_BUILD_DIR}/arrow
${OFFLINE_BUILD_DIR}/arrow/cpp/thirdparty/download_dependencies.sh ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty
wget https://github.com/substrait-io/substrait/archive/v${SUBSTRAIT_VERSION}.tar.gz -O ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty/substrait-${SUBSTRAIT_VERSION}.tar.gz

alias cachi2='docker run --rm -ti -v "$PWD:$PWD:z" -w "$PWD" quay.io/konflux-ci/cachi2:f7a61b067f4446e4982d0e3b9545ce4aa0d8284f'
cachi2 fetch-deps \
--output ${OFFLINE_BUILD_DIR}/cachi2-output \
alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
# not needed for downstream build from release
###############################
hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui-output \
'{
"type": "yarn",
"path": "ui"
}'
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui.env --for-output-dir /tmp/hermeto-yarn-ui-output ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui-output

hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-yarn-output \
'{
"type": "yarn",
"path": "sdk/python/feast/ui"
}'
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto-yarn.env --for-output-dir /tmp/hermeto-yarn-output ${OFFLINE_BUILD_DIR}/hermeto-yarn-output
###############################

hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-output \
'{
"type": "pip",
"path": ".",
"requirements_files": [
"sdk/python/feast/infra/feature_servers/multicloud/requirements.txt"
],
"requirements_build_files": [
"sdk/python/feast/infra/feature_servers/multicloud/offline/pyarrow17-wheel-build-requirements.txt",
"sdk/python/feast/infra/feature_servers/multicloud/offline/psycopg3.2.5-wheel-build-requirements.txt",
"sdk/python/requirements/py3.11-sdist-requirements.txt",
"sdk/python/requirements/py3.11-pandas-requirements.txt",
"sdk/python/requirements/py3.11-addtl-sources-requirements.txt"
"sdk/python/requirements/py3.11-minimal-sdist-requirements.txt",
"sdk/python/requirements/py3.11-minimal-sdist-requirements-build.txt"
],
"allow_binary": "false"
}'
cachi2 generate-env ${OFFLINE_BUILD_DIR}/cachi2-output -o ${OFFLINE_BUILD_DIR}/cachi2.env --for-output-dir /tmp/cachi2-output
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto.env --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output
hermeto inject-files --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output

# arrow OFFLINE builder - version 17.0.0
# arrow OFFLINE builder
rm -f ${OFFLINE_BUILD_DIR}/arrow/.dockerignore
docker build \
--volume ${OFFLINE_BUILD_DIR}/arrow:/tmp/arrow:Z \
--volume ${OFFLINE_BUILD_DIR}/cachi2-output:/tmp/cachi2-output:Z \
--volume ${OFFLINE_BUILD_DIR}/cachi2.env:/tmp/cachi2.env:Z \
--volume ${PROJECT_ROOT_DIR}/sdk/python/feast/infra/feature_servers/multicloud/offline:/tmp/offline:ro \
--network none \
--volume ${OFFLINE_BUILD_DIR}/arrow:/tmp/arrow:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
--tag arrow-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.arrow \
--load offline_build/arrow
${OFFLINE_BUILD_DIR}/arrow

# pip builder
# ibis OFFLINE builder - ibis-framework must build from its own git repo... versioning requirement
git clone -b ${IBIS_VERSION} https://github.com/ibis-project/ibis ${OFFLINE_BUILD_DIR}/ibis
docker build \
--tag pip-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.pip \
--load sdk/python/feast/infra/feature_servers/multicloud/offline

# ibis OFFLINE builder
docker build \
--volume ${OFFLINE_BUILD_DIR}/cachi2-output:/tmp/cachi2-output:Z \
--volume ${OFFLINE_BUILD_DIR}/cachi2.env:/tmp/cachi2.env:Z \
--network none \
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
--tag ibis-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.ibis \
--load sdk/python/feast/infra/feature_servers/multicloud/offline

# is this needed? check for reqs logs as feast builds
# maturin OFFLINE builder
#mkdir -p ${OFFLINE_BUILD_DIR}/cachi2-maturin
#git clone --branch v1.8.3 https://github.com/PyO3/maturin ${OFFLINE_BUILD_DIR}/maturin
#cachi2 fetch-deps cargo --source ${OFFLINE_BUILD_DIR}/maturin --output ${OFFLINE_BUILD_DIR}/cachi2-maturin
#cachi2 inject-files --for-output-dir /tmp/cachi2-maturin ${OFFLINE_BUILD_DIR}/cachi2-maturin
#docker build \
# --volume ${OFFLINE_BUILD_DIR}/cachi2-maturin:/tmp/cachi2-maturin:Z \
# --volume ${OFFLINE_BUILD_DIR}/cachi2-output:/tmp/cachi2-output:Z \
# --volume ${OFFLINE_BUILD_DIR}/cachi2.env:/tmp/cachi2.env:Z \
# --network none \
# --tag maturin-builder \
# -f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.maturin \
# --load ${OFFLINE_BUILD_DIR}/maturin
${OFFLINE_BUILD_DIR}

# feast OFFLINE builder
docker build \
--volume ${OFFLINE_BUILD_DIR}/cachi2-output:/tmp/cachi2-output:Z \
--volume ${OFFLINE_BUILD_DIR}/cachi2.env:/tmp/cachi2.env:Z \
--network none \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui-output:/tmp/hermeto-yarn-ui-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn-ui.env:/tmp/hermeto-yarn-ui.env:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn-output:/tmp/hermeto-yarn-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-yarn.env:/tmp/hermeto-yarn.env:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
--tag feature-server:sdist-build \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist \
--load sdk/python/feast/infra/feature_servers/multicloud
${PROJECT_ROOT_DIR}

docker run --rm -ti feature-server:sdist-build feast version
58 changes: 58 additions & 0 deletions infra/scripts/offline-release-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# to run -> source ./infra/scripts/offline-release-build.sh
# on the build host... requires docker/podman, git, wget

APACHE_ARROW_VERSION="17.0.0"
SUBSTRAIT_VERSION="0.44.0"
IBIS_VERSION="9.5.0"

# Get Feast project repository root directory
PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
OFFLINE_BUILD_DIR=${PROJECT_ROOT_DIR}/offline_build
cd ${PROJECT_ROOT_DIR}

rm -rf ./offline_build
mkdir offline_build

# yum builder
docker build \
--build-arg RELEASE=true \
--tag yum-builder \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.builder.yum \
sdk/python/feast/infra/feature_servers/multicloud/offline

git clone --branch apache-arrow-${APACHE_ARROW_VERSION} https://github.com/apache/arrow ${OFFLINE_BUILD_DIR}/arrow
${OFFLINE_BUILD_DIR}/arrow/cpp/thirdparty/download_dependencies.sh ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty
wget https://github.com/substrait-io/substrait/archive/v${SUBSTRAIT_VERSION}.tar.gz -O ${OFFLINE_BUILD_DIR}/arrow/cpp/arrow-thirdparty/substrait-${SUBSTRAIT_VERSION}.tar.gz

alias hermeto='docker run --rm -ti -v "$PWD:$PWD:Z" -w "$PWD" quay.io/konflux-ci/hermeto:0.24.0'
hermeto fetch-deps \
--output ${OFFLINE_BUILD_DIR}/hermeto-output \
'{
"type": "pip",
"path": ".",
"requirements_files": [
"sdk/python/feast/infra/feature_servers/multicloud/requirements.txt"
],
"requirements_build_files": [
"sdk/python/requirements/py3.11-minimal-sdist-requirements.txt",
"sdk/python/requirements/py3.11-minimal-sdist-requirements-build.txt"
],
"allow_binary": "false"
}'
hermeto generate-env -o ${OFFLINE_BUILD_DIR}/hermeto.env --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output
hermeto inject-files --for-output-dir /tmp/hermeto-output ${OFFLINE_BUILD_DIR}/hermeto-output

# ibis OFFLINE builder - ibis-framework must build from its own git repo... versioning requirement
git clone -b ${IBIS_VERSION} https://github.com/ibis-project/ibis ${OFFLINE_BUILD_DIR}/ibis

# feast OFFLINE builder
docker build \
--network none \
--volume ${OFFLINE_BUILD_DIR}/arrow:/tmp/arrow:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto-output:/tmp/hermeto-output:Z \
--volume ${OFFLINE_BUILD_DIR}/hermeto.env:/tmp/hermeto.env:Z \
--tag feature-server:sdist-release-build \
-f sdk/python/feast/infra/feature_servers/multicloud/offline/Dockerfile.sdist.release \
${PROJECT_ROOT_DIR}

docker run --rm -ti feature-server:sdist-release-build feast version
Loading
Loading