Skip to content

Commit bcdce32

Browse files
pyalexpyalex
authored
chore: Test all wheels for basic functionality before publishing to pypi (#2687)
* verify python wheel before publish Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com> * revert debug Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
1 parent 4f85e3e commit bcdce32

2 files changed

Lines changed: 135 additions & 65 deletions

File tree

‎.github/workflows/publish.yml‎

Lines changed: 58 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -140,23 +140,71 @@ jobs:
140140

141141

142142
verify-python-wheel:
143-
runs-on: ubuntu-latest
143+
runs-on: ${{ matrix.os }}
144144
needs: [build-python-sdk, build-python-sdk-macos-py310]
145+
strategy:
146+
matrix:
147+
os: [ ubuntu-latest, macos-10.15 ]
148+
python-version: [ "3.7", "3.8", "3.9", "3.10"]
149+
from-source: [ True, False ]
150+
env:
151+
# this script is for testing servers
152+
# it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
153+
TEST_SCRIPT: |
154+
timeout 10s $@ & pid=$!
155+
wait $pid
156+
ret=$?
157+
if [[ $ret -ne 124 ]]
158+
then
159+
exit $ret
160+
else
161+
echo "Succeeded!"
162+
fi
145163
steps:
146164
- name: Setup Python
147165
id: setup-python
148166
uses: actions/setup-python@v2
149167
with:
150-
python-version: "3.8"
168+
python-version: ${{ matrix.python-version }}
151169
architecture: x64
170+
- uses: actions/setup-go@v3
171+
with:
172+
go-version: '>=1.17.0'
152173
- uses: actions/download-artifact@v2
153174
with:
154175
name: wheels
155176
path: dist
156177
- name: Install wheel
157-
run: pip install dist/*38*linux*.whl
178+
if: ${{ !matrix.from-source }}
179+
# try to install all wheels; only the current platform wheel should be actually installed
180+
run: |
181+
cd dist/
182+
pip install wheel
183+
for f in *.whl; do pip install $f || true; done
158184
- name: Install sdist
159-
run: pip install dist/*tar.gz
185+
if: ${{ matrix.from-source }}
186+
env:
187+
COMPILE_GO: "True"
188+
run: |
189+
pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
190+
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
191+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
192+
pip install dist/*tar.gz
193+
- name: Install OS X dependencies
194+
if: matrix.os == 'macos-10.15'
195+
run: brew install coreutils
196+
- name: Smoke test
197+
run: |
198+
feast init test_repo
199+
cd test_repo/
200+
feast apply
201+
echo "$TEST_SCRIPT" > run-and-wait.sh
202+
bash run-and-wait.sh feast serve
203+
bash run-and-wait.sh feast ui
204+
205+
pip install cffi
206+
printf "\ngo_feature_retrieval: True" >> feature_store.yaml
207+
bash run-and-wait.sh feast serve
160208
161209
build-python-sdk:
162210
name: Build wheels on ${{ matrix.os }}
@@ -171,24 +219,27 @@ jobs:
171219
with:
172220
node-version: '17.x'
173221
registry-url: 'https://registry.npmjs.org'
222+
- name: Build UI
223+
run: make build-ui
174224
- name: Build wheels
175225
uses: pypa/cibuildwheel@v2.4.0
176226
env:
177227
CIBW_BUILD: "cp3*_x86_64"
178228
CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
179229
CIBW_ARCHS: "native"
180230
CIBW_ENVIRONMENT: >
181-
COMPILE_GO=True
231+
COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
182232
CIBW_BEFORE_ALL_LINUX: |
183-
yum install -y golang
233+
curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
234+
tar -C /usr/local -xzf go.tar.gz
235+
go version
184236
CIBW_BEFORE_ALL_MACOS: |
185237
curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
186238
sudo installer -pkg python.pkg -target /
187239
CIBW_BEFORE_BUILD: |
188240
make install-protoc-dependencies
189241
make install-go-proto-dependencies
190242
make install-go-ci-dependencies
191-
make build-ui
192243
193244
- uses: actions/upload-artifact@v2
194245
with:

‎setup.py‎

Lines changed: 77 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,7 @@
114114
"happybase>=1.2.0,<3",
115115
]
116116

117-
GE_REQUIRED = [
118-
"great_expectations>=0.14.0,<0.15.0"
119-
]
117+
GE_REQUIRED = ["great_expectations>=0.14.0,<0.15.0"]
120118

121119
GO_REQUIRED = [
122120
"cffi==1.15.*,<2",
@@ -166,15 +164,15 @@
166164
"types-setuptools",
167165
"types-tabulate",
168166
]
169-
+ GCP_REQUIRED
170-
+ REDIS_REQUIRED
171-
+ AWS_REQUIRED
172-
+ SNOWFLAKE_REQUIRED
173-
+ SPARK_REQUIRED
174-
+ POSTGRES_REQUIRED
175-
+ TRINO_REQUIRED
176-
+ GE_REQUIRED
177-
+ HBASE_REQUIRED
167+
+ GCP_REQUIRED
168+
+ REDIS_REQUIRED
169+
+ AWS_REQUIRED
170+
+ SNOWFLAKE_REQUIRED
171+
+ SPARK_REQUIRED
172+
+ POSTGRES_REQUIRED
173+
+ TRINO_REQUIRED
174+
+ GE_REQUIRED
175+
+ HBASE_REQUIRED
178176
)
179177

180178
DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED
@@ -206,7 +204,9 @@
206204

207205
class BuildPythonProtosCommand(Command):
208206
description = "Builds the proto files into Python files."
209-
user_options = []
207+
user_options = [
208+
("inplace", "i", "Write generated proto files to source directory."),
209+
]
210210

211211
def initialize_options(self):
212212
self.python_protoc = [
@@ -215,13 +215,21 @@ def initialize_options(self):
215215
"grpc_tools.protoc",
216216
] # find_executable("protoc")
217217
self.proto_folder = os.path.join(repo_root, "protos")
218-
self.python_folder = os.path.join(
219-
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
220-
)
221218
self.sub_folders = PROTO_SUBDIRS
219+
self.build_lib = None
220+
self.inplace = 0
222221

223222
def finalize_options(self):
224-
pass
223+
self.set_undefined_options("build", ("build_lib", "build_lib"))
224+
225+
@property
226+
def python_folder(self):
227+
if self.inplace:
228+
return os.path.join(
229+
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
230+
)
231+
232+
return os.path.join(self.build_lib, "feast/protos")
225233

226234
def _generate_python_protos(self, path: str):
227235
proto_files = glob.glob(os.path.join(self.proto_folder, path))
@@ -247,13 +255,12 @@ def run(self):
247255
# We need the __init__ files for each of the generated subdirs
248256
# so that they are regular packages, and don't need the `--namespace-packages` flags
249257
# when being typechecked using mypy.
250-
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'):
258+
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", "w"):
251259
pass
252260

253-
254-
with open(f"{self.python_folder}/__init__.py", 'w'):
261+
with open(f"{self.python_folder}/__init__.py", "w"):
255262
pass
256-
with open(f"{self.python_folder}/feast/__init__.py", 'w'):
263+
with open(f"{self.python_folder}/feast/__init__.py", "w"):
257264
pass
258265

259266
for path in Path(self.python_folder).rglob("*.py"):
@@ -295,12 +302,10 @@ def _ensure_go_and_proto_toolchain():
295302
path_val = _generate_path_with_gopath()
296303

297304
try:
298-
subprocess.check_call(["protoc-gen-go", "--version"], env={
299-
"PATH": path_val
300-
})
301-
subprocess.check_call(["protoc-gen-go-grpc", "--version"], env={
302-
"PATH": path_val
303-
})
305+
subprocess.check_call(["protoc-gen-go", "--version"], env={"PATH": path_val})
306+
subprocess.check_call(
307+
["protoc-gen-go-grpc", "--version"], env={"PATH": path_val}
308+
)
304309
except Exception as e:
305310
raise RuntimeError("Unable to find go/grpc extensions for protoc") from e
306311

@@ -329,15 +334,18 @@ def _generate_go_protos(self, path: str):
329334
try:
330335
subprocess.check_call(
331336
self.go_protoc
332-
+ ["-I", self.proto_folder,
333-
"--go_out", self.go_folder,
334-
"--go_opt=module=github.com/feast-dev/feast/go/protos",
335-
"--go-grpc_out", self.go_folder,
336-
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos"]
337+
+ [
338+
"-I",
339+
self.proto_folder,
340+
"--go_out",
341+
self.go_folder,
342+
"--go_opt=module=github.com/feast-dev/feast/go/protos",
343+
"--go-grpc_out",
344+
self.go_folder,
345+
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos",
346+
]
337347
+ proto_files,
338-
env={
339-
"PATH": self.path_val
340-
}
348+
env={"PATH": self.path_val},
341349
)
342350
except CalledProcessError as e:
343351
print(f"Stderr: {e.stderr}")
@@ -355,18 +363,19 @@ class BuildCommand(build_py):
355363

356364
def run(self):
357365
self.run_command("build_python_protos")
358-
self.run_command("build_ext")
359366
if os.getenv("COMPILE_GO", "false").lower() == "true":
360367
_ensure_go_and_proto_toolchain()
361368
self.run_command("build_go_protos")
362369

370+
self.run_command("build_ext")
363371
build_py.run(self)
364372

365373

366374
class DevelopCommand(develop):
367375
"""Custom develop command."""
368376

369377
def run(self):
378+
self.reinitialize_command("build_python_protos", inplace=1)
370379
self.run_command("build_python_protos")
371380
if os.getenv("COMPILE_GO", "false").lower() == "true":
372381
_ensure_go_and_proto_toolchain()
@@ -382,7 +391,10 @@ def finalize_options(self) -> None:
382391
self.extensions = [e for e in self.extensions if not self._is_go_ext(e)]
383392

384393
def _is_go_ext(self, ext: Extension):
385-
return any(source.endswith('.go') or source.startswith('github') for source in ext.sources)
394+
return any(
395+
source.endswith(".go") or source.startswith("github")
396+
for source in ext.sources
397+
)
386398

387399
def build_extension(self, ext: Extension):
388400
if not self._is_go_ext(ext):
@@ -400,33 +412,34 @@ def build_extension(self, ext: Extension):
400412
)
401413

402414
destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name)))
403-
subprocess.check_call([
404-
"gopy",
405-
"build",
406-
"-output",
407-
destination,
408-
"-vm",
409-
sys.executable,
410-
"-no-make",
411-
*ext.sources
412-
], env={
413-
"PATH": bin_path,
414-
"CGO_LDFLAGS_ALLOW": ".*",
415-
**go_env,
416-
})
415+
subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"])
416+
subprocess.check_call(["go", "install", "github.com/go-python/gopy"])
417+
subprocess.check_call(
418+
[
419+
"gopy",
420+
"build",
421+
"-output",
422+
destination,
423+
"-vm",
424+
sys.executable,
425+
"-no-make",
426+
*ext.sources,
427+
],
428+
env={"PATH": bin_path, "CGO_LDFLAGS_ALLOW": ".*", **go_env,},
429+
)
417430

418431
def copy_extensions_to_source(self):
419-
build_py = self.get_finalized_command('build_py')
432+
build_py = self.get_finalized_command("build_py")
420433
for ext in self.extensions:
421434
fullname = self.get_ext_fullname(ext.name)
422-
modpath = fullname.split('.')
423-
package = '.'.join(modpath[:-1])
435+
modpath = fullname.split(".")
436+
package = ".".join(modpath[:-1])
424437
package_dir = build_py.get_package_dir(package)
425438

426439
src_dir = dest_dir = package_dir
427440

428441
if src_dir.startswith(PYTHON_CODE_PREFIX):
429-
src_dir = package_dir[len(PYTHON_CODE_PREFIX):]
442+
src_dir = package_dir[len(PYTHON_CODE_PREFIX) :]
430443
src_dir = src_dir.lstrip("/")
431444

432445
src_dir = os.path.join(self.build_lib, src_dir)
@@ -443,7 +456,9 @@ def copy_extensions_to_source(self):
443456
long_description_content_type="text/markdown",
444457
python_requires=REQUIRES_PYTHON,
445458
url=URL,
446-
packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")),
459+
packages=find_packages(
460+
where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")
461+
),
447462
package_dir={"": PYTHON_CODE_PREFIX},
448463
install_requires=REQUIRED,
449464
# https://stackoverflow.com/questions/28509965/setuptools-development-requirements
@@ -488,6 +503,10 @@ def copy_extensions_to_source(self):
488503
"develop": DevelopCommand,
489504
"build_ext": build_ext,
490505
},
491-
ext_modules=[Extension('feast.embedded_go.lib._embedded',
492-
["github.com/feast-dev/feast/go/embedded"])],
506+
ext_modules=[
507+
Extension(
508+
"feast.embedded_go.lib._embedded",
509+
["github.com/feast-dev/feast/go/embedded"],
510+
)
511+
],
493512
)

0 commit comments

Comments
 (0)