Skip to content

chore: Test all wheels for basic functionality before publishing to pypi #2687

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 2 commits into from
May 13, 2022
Merged

chore: Test all wheels for basic functionality before publishing to pypi #2687

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4eeab07
verify python wheel before publish
pyalex May 11, 2022
301c91f
revert debug
pyalex May 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 58 additions & 7 deletions .github/workflows/publish.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,23 +140,71 @@ jobs:


verify-python-wheel:
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
needs: [build-python-sdk, build-python-sdk-macos-py310]
strategy:
matrix:
os: [ ubuntu-latest, macos-10.15 ]
python-version: [ "3.7", "3.8", "3.9", "3.10"]
from-source: [ True, False ]
env:
# this script is for testing servers
# it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
TEST_SCRIPT: |
timeout 10s $@ & pid=$!
wait $pid
ret=$?
if [[ $ret -ne 124 ]]
then
exit $ret
else
echo "Succeeded!"
fi
steps:
- name: Setup Python
id: setup-python
uses: actions/setup-python@v2
with:
python-version: "3.8"
python-version: ${{ matrix.python-version }}
architecture: x64
- uses: actions/setup-go@v3
with:
go-version: '>=1.17.0'
- uses: actions/download-artifact@v2
with:
name: wheels
path: dist
- name: Install wheel
run: pip install dist/*38*linux*.whl
if: ${{ !matrix.from-source }}
# try to install all wheels; only the current platform wheel should be actually installed
run: |
cd dist/
pip install wheel
for f in *.whl; do pip install $f || true; done
- name: Install sdist
run: pip install dist/*tar.gz
if: ${{ matrix.from-source }}
env:
COMPILE_GO: "True"
run: |
pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
pip install dist/*tar.gz
- name: Install OS X dependencies
if: matrix.os == 'macos-10.15'
run: brew install coreutils
- name: Smoke test
run: |
feast init test_repo
cd test_repo/
feast apply
echo "$TEST_SCRIPT" > run-and-wait.sh
bash run-and-wait.sh feast serve
bash run-and-wait.sh feast ui

pip install cffi
printf "\ngo_feature_retrieval: True" >> feature_store.yaml
bash run-and-wait.sh feast serve

build-python-sdk:
name: Build wheels on ${{ matrix.os }}
Expand All @@ -171,24 +219,27 @@ jobs:
with:
node-version: '17.x'
registry-url: 'https://registry.npmjs.org'
- name: Build UI
run: make build-ui
- name: Build wheels
uses: pypa/cibuildwheel@v2.4.0
env:
CIBW_BUILD: "cp3*_x86_64"
CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
CIBW_ARCHS: "native"
CIBW_ENVIRONMENT: >
COMPILE_GO=True
COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
CIBW_BEFORE_ALL_LINUX: |
yum install -y golang
curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
tar -C /usr/local -xzf go.tar.gz
go version
CIBW_BEFORE_ALL_MACOS: |
curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
sudo installer -pkg python.pkg -target /
CIBW_BEFORE_BUILD: |
make install-protoc-dependencies
make install-go-proto-dependencies
make install-go-ci-dependencies
make build-ui

- uses: actions/upload-artifact@v2
with:
Expand Down
135 changes: 77 additions & 58 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,7 @@
"happybase>=1.2.0,<3",
]

GE_REQUIRED = [
"great_expectations>=0.14.0,<0.15.0"
]
GE_REQUIRED = ["great_expectations>=0.14.0,<0.15.0"]

GO_REQUIRED = [
"cffi==1.15.*,<2",
Expand Down Expand Up @@ -166,15 +164,15 @@
"types-setuptools",
"types-tabulate",
]
+ GCP_REQUIRED
+ REDIS_REQUIRED
+ AWS_REQUIRED
+ SNOWFLAKE_REQUIRED
+ SPARK_REQUIRED
+ POSTGRES_REQUIRED
+ TRINO_REQUIRED
+ GE_REQUIRED
+ HBASE_REQUIRED
+ GCP_REQUIRED
+ REDIS_REQUIRED
+ AWS_REQUIRED
+ SNOWFLAKE_REQUIRED
+ SPARK_REQUIRED
+ POSTGRES_REQUIRED
+ TRINO_REQUIRED
+ GE_REQUIRED
+ HBASE_REQUIRED
)

DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED
Expand Down Expand Up @@ -206,7 +204,9 @@

class BuildPythonProtosCommand(Command):
description = "Builds the proto files into Python files."
user_options = []
user_options = [
("inplace", "i", "Write generated proto files to source directory."),
]

def initialize_options(self):
self.python_protoc = [
Expand All @@ -215,13 +215,21 @@ def initialize_options(self):
"grpc_tools.protoc",
] # find_executable("protoc")
self.proto_folder = os.path.join(repo_root, "protos")
self.python_folder = os.path.join(
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
)
self.sub_folders = PROTO_SUBDIRS
self.build_lib = None
self.inplace = 0

def finalize_options(self):
pass
self.set_undefined_options("build", ("build_lib", "build_lib"))

@property
def python_folder(self):
if self.inplace:
return os.path.join(
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
)

return os.path.join(self.build_lib, "feast/protos")

def _generate_python_protos(self, path: str):
proto_files = glob.glob(os.path.join(self.proto_folder, path))
Expand All @@ -247,13 +255,12 @@ def run(self):
# We need the __init__ files for each of the generated subdirs
# so that they are regular packages, and don't need the `--namespace-packages` flags
# when being typechecked using mypy.
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'):
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", "w"):
pass


with open(f"{self.python_folder}/__init__.py", 'w'):
with open(f"{self.python_folder}/__init__.py", "w"):
pass
with open(f"{self.python_folder}/feast/__init__.py", 'w'):
with open(f"{self.python_folder}/feast/__init__.py", "w"):
pass

for path in Path(self.python_folder).rglob("*.py"):
Expand Down Expand Up @@ -295,12 +302,10 @@ def _ensure_go_and_proto_toolchain():
path_val = _generate_path_with_gopath()

try:
subprocess.check_call(["protoc-gen-go", "--version"], env={
"PATH": path_val
})
subprocess.check_call(["protoc-gen-go-grpc", "--version"], env={
"PATH": path_val
})
subprocess.check_call(["protoc-gen-go", "--version"], env={"PATH": path_val})
subprocess.check_call(
["protoc-gen-go-grpc", "--version"], env={"PATH": path_val}
)
except Exception as e:
raise RuntimeError("Unable to find go/grpc extensions for protoc") from e

Expand Down Expand Up @@ -329,15 +334,18 @@ def _generate_go_protos(self, path: str):
try:
subprocess.check_call(
self.go_protoc
+ ["-I", self.proto_folder,
"--go_out", self.go_folder,
"--go_opt=module=github.com/feast-dev/feast/go/protos",
"--go-grpc_out", self.go_folder,
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos"]
+ [
"-I",
self.proto_folder,
"--go_out",
self.go_folder,
"--go_opt=module=github.com/feast-dev/feast/go/protos",
"--go-grpc_out",
self.go_folder,
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos",
]
+ proto_files,
env={
"PATH": self.path_val
}
env={"PATH": self.path_val},
)
except CalledProcessError as e:
print(f"Stderr: {e.stderr}")
Expand All @@ -355,18 +363,19 @@ class BuildCommand(build_py):

def run(self):
self.run_command("build_python_protos")
self.run_command("build_ext")
if os.getenv("COMPILE_GO", "false").lower() == "true":
_ensure_go_and_proto_toolchain()
self.run_command("build_go_protos")

self.run_command("build_ext")
build_py.run(self)


class DevelopCommand(develop):
"""Custom develop command."""

def run(self):
self.reinitialize_command("build_python_protos", inplace=1)
self.run_command("build_python_protos")
if os.getenv("COMPILE_GO", "false").lower() == "true":
_ensure_go_and_proto_toolchain()
Expand All @@ -382,7 +391,10 @@ def finalize_options(self) -> None:
self.extensions = [e for e in self.extensions if not self._is_go_ext(e)]

def _is_go_ext(self, ext: Extension):
return any(source.endswith('.go') or source.startswith('github') for source in ext.sources)
return any(
source.endswith(".go") or source.startswith("github")
for source in ext.sources
)

def build_extension(self, ext: Extension):
if not self._is_go_ext(ext):
Expand All @@ -400,33 +412,34 @@ def build_extension(self, ext: Extension):
)

destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name)))
subprocess.check_call([
"gopy",
"build",
"-output",
destination,
"-vm",
sys.executable,
"-no-make",
*ext.sources
], env={
"PATH": bin_path,
"CGO_LDFLAGS_ALLOW": ".*",
**go_env,
})
subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"])
subprocess.check_call(["go", "install", "github.com/go-python/gopy"])
subprocess.check_call(
[
"gopy",
"build",
"-output",
destination,
"-vm",
sys.executable,
"-no-make",
*ext.sources,
],
env={"PATH": bin_path, "CGO_LDFLAGS_ALLOW": ".*", **go_env,},
)

def copy_extensions_to_source(self):
build_py = self.get_finalized_command('build_py')
build_py = self.get_finalized_command("build_py")
for ext in self.extensions:
fullname = self.get_ext_fullname(ext.name)
modpath = fullname.split('.')
package = '.'.join(modpath[:-1])
modpath = fullname.split(".")
package = ".".join(modpath[:-1])
package_dir = build_py.get_package_dir(package)

src_dir = dest_dir = package_dir

if src_dir.startswith(PYTHON_CODE_PREFIX):
src_dir = package_dir[len(PYTHON_CODE_PREFIX):]
src_dir = package_dir[len(PYTHON_CODE_PREFIX) :]
src_dir = src_dir.lstrip("/")

src_dir = os.path.join(self.build_lib, src_dir)
Expand All @@ -443,7 +456,9 @@ def copy_extensions_to_source(self):
long_description_content_type="text/markdown",
python_requires=REQUIRES_PYTHON,
url=URL,
packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")),
packages=find_packages(
where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")
),
package_dir={"": PYTHON_CODE_PREFIX},
install_requires=REQUIRED,
# https://stackoverflow.com/questions/28509965/setuptools-development-requirements
Expand Down Expand Up @@ -488,6 +503,10 @@ def copy_extensions_to_source(self):
"develop": DevelopCommand,
"build_ext": build_ext,
},
ext_modules=[Extension('feast.embedded_go.lib._embedded',
["github.com/feast-dev/feast/go/embedded"])],
ext_modules=[
Extension(
"feast.embedded_go.lib._embedded",
["github.com/feast-dev/feast/go/embedded"],
)
],
)