Skip to content

Commit 0c8a3b0

Browse files
cztomczakcztomczak
committed
Reduce packages size with build_distrib.py tool (cztomczak#321, cztomczak#262)...
Test wheel package installation in build_distrib.py. Add --no-rebuild flag to build_distrib.py.
1 parent 2882865 commit 0c8a3b0

File tree

4 files changed

+132
-38
lines changed

4 files changed

+132
-38
lines changed

‎src/compile_time_constants.pxi‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# This file was generated by setup.py
2-
DEF UNAME_SYSNAME = "Linux"
3-
DEF PY_MAJOR_VERSION = 2
2+
DEF UNAME_SYSNAME = "Windows"
3+
DEF PY_MAJOR_VERSION = 3

‎tools/automate.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ def build_cef_projects():
348348
"build_cefclient")
349349
cefclient_exe = os.path.join(build_cefclient_dir, "tests", "cefclient",
350350
Options.build_type,
351-
"cefclient" + EXECUTABLE_EXT)
351+
"cefclient" + APP_EXT)
352352

353353
# Check whether already built
354354
already_built = False
@@ -691,7 +691,7 @@ def create_prebuilt_binaries():
691691
src,
692692
"build_cefclient", "tests", "cefclient",
693693
Options.build_type,
694-
"cefclient" + EXECUTABLE_EXT)
694+
"cefclient" + APP_EXT)
695695
if LINUX and os.path.exists(cefclient):
696696
# On Windows resources/*.html files are embedded inside exe
697697
cefclient_files = os.path.join(
@@ -706,14 +706,14 @@ def create_prebuilt_binaries():
706706
src,
707707
"build_cefclient", "tests", "cefsimple",
708708
Options.build_type,
709-
"cefsimple" + EXECUTABLE_EXT)
709+
"cefsimple" + APP_EXT)
710710

711711
# ceftests
712712
ceftests = os.path.join(
713713
src,
714714
"build_cefclient", "tests", "ceftests",
715715
Options.build_type,
716-
"ceftests" + EXECUTABLE_EXT)
716+
"ceftests" + APP_EXT)
717717
if LINUX and os.path.exists(ceftests):
718718
# On Windows resources/*.html files are embedded inside exe
719719
ceftests_files = os.path.join(

‎tools/build_distrib.py‎

Lines changed: 116 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,18 @@
66
Build distribution packages for all architectures and all supported
77
python versions.
88
9-
TODO: test_wheel_package_installation()
109
TODO: Linux/Mac support. Currently runs only on Windows.
1110
1211
Usage:
13-
build_distrib.py VERSION [--no-run-examples]
12+
build_distrib.py VERSION [--no-run-examples] [--no-rebuild]
1413
1514
Options:
1615
VERSION Version number eg. 50.0
1716
--no-run-examples Do not run examples while building cefpython modules.
1817
Only unit tests will be run in such case.
18+
--no-rebuild Do not rebuild cefpython modules. For internal use
19+
so that changes to packaging can be quickly tested.
20+
1921
2022
This script does the following:
2123
1. Expects that all supported python versions are installed
@@ -27,17 +29,24 @@
2729
2. Expects that all python compilers for supported python versions
2830
are installed. See docs/Build-instructions.md > Requirements.
2931
3. Expects cef_binary*/ directories from Spotify Automated Builds
30-
to be in the build/ directory
32+
to be in the build/ directory. It does not rebuild cefclient
33+
nor libcef_dll_wrapper libraries in these directories. If you
34+
would like to rebuild everything from scratch then delete subdirs
35+
manually (build_cefclient/, build_wrapper*/).
3136
4. Install and/or upgrade tools/requirements.txt and uninstall
3237
cefpython3 packages for all python versions
3338
5. Run automate.py --prebuilt-cef using both Python 32-bit and Python 64-bit
3439
6. Pack the prebuilt biaries using zip on Win/Mac and .tar.gz on Linux
3540
and move to build/distrib/
36-
7. Build cefpython modules for all supported Python versions on both
41+
7. Reduce packages size (Issue #321). After packing prebuilt binaries,
42+
reduce its size so that packages will use the reduced prebuilt binaries.
43+
8. Build cefpython modules for all supported Python versions on both
3744
32-bit and 64-bit
38-
8. Make setup installers and pack them to zip (Win/Mac) or .tar.gz (Linux)
39-
9. Make wheel packages
40-
10. Move setup and wheel packages to the build/distrib/ directory
45+
9. Make setup installers and pack them to zip (Win/Mac) or .tar.gz (Linux)
46+
10. Make wheel packages
47+
11. Move setup and wheel packages to the build/distrib/ directory
48+
12. Test installation of wheel packages
49+
13. Show summary
4150
"""
4251

4352
from common import *
@@ -52,6 +61,7 @@
5261
# Command line args
5362
VERSION = ""
5463
NO_RUN_EXAMPLES = False
64+
NO_REBUILD = False
5565

5666
# Pythons
5767
SUPPORTED_PYTHON_VERSIONS = [(2, 7), (3, 4), (3, 5), (3, 6)]
@@ -81,50 +91,67 @@ def main():
8191
os.makedirs(DISTRIB_DIR)
8292
if pythons_32bit:
8393
run_automate_prebuilt_cef(pythons_32bit[0])
84-
pack_prebuilt_cef(pythons_32bit[0]["arch"])
94+
pack_prebuilt_cef("32bit")
95+
if LINUX:
96+
reduce_package_size_issue_262("32bit")
97+
reduce_package_size_issue_321("32bit")
8598
if pythons_64bit is not None:
8699
run_automate_prebuilt_cef(pythons_64bit[0])
87-
pack_prebuilt_cef(pythons_64bit[0]["arch"])
88-
build_cefpython_modules(pythons_32bit + pythons_64bit)
100+
pack_prebuilt_cef("64bit")
101+
if LINUX:
102+
reduce_package_size_issue_262("64bit")
103+
reduce_package_size_issue_321("64bit")
104+
if not NO_REBUILD:
105+
build_cefpython_modules(pythons_32bit + pythons_64bit)
89106
if pythons_32bit:
90107
make_packages(pythons_32bit[0], "32bit")
91108
if pythons_64bit:
92109
make_packages(pythons_64bit[0], "64bit")
93-
test_wheel_package_installation()
110+
test_wheel_package_installation(pythons_32bit + pythons_64bit)
94111
show_summary(pythons_32bit, pythons_64bit)
95112

96113

97114
def command_line_args():
98-
global VERSION, NO_RUN_EXAMPLES
115+
global VERSION, NO_RUN_EXAMPLES, NO_REBUILD
99116
version = get_version_from_command_line_args(__file__)
100-
if not version:
117+
if not version or "--help" in sys.argv:
101118
print(__doc__)
102119
sys.exit(1)
103120
VERSION = version
104121
if "--no-run-examples" in sys.argv:
105122
NO_RUN_EXAMPLES = True
123+
sys.argv.remove("--no-run-examples")
124+
if "--no-rebuild" in sys.argv:
125+
NO_REBUILD = True
126+
sys.argv.remove("--no-rebuild")
127+
args = sys.argv[1:]
128+
for arg in args:
129+
if arg == version:
130+
continue
131+
print("[build_distrib.py] Invalid argument: {arg}".format(arg=arg))
132+
sys.exit(1)
106133

107134

108135
def clean_build_directories():
109136
print("[build_distrib.py] Clean build directories")
110137

111-
# Distrib dir
138+
# Delete distrib dir
112139
if os.path.exists(DISTRIB_DIR):
113140
print("[build_distrib.py] Delete directory: {distrib_dir}/"
114141
.format(distrib_dir=os.path.basename(DISTRIB_DIR)))
115142
shutil.rmtree(DISTRIB_DIR)
116143

117-
# build_cefpython/ dir
118-
if os.path.exists(BUILD_CEFPYTHON):
119-
print("[build_distirb.py] Delete directory: {dir}/"
120-
.format(dir=os.path.basename(BUILD_CEFPYTHON)))
121-
shutil.rmtree(BUILD_CEFPYTHON)
122-
123-
# cefpython_binary_*/ dirs
124-
delete_cefpython_binary_dir("32bit")
125-
delete_cefpython_binary_dir("64bit")
126-
127-
# cef binaries and libraries dirs
144+
if not NO_REBUILD:
145+
# Delete build_cefpython/ dir
146+
if os.path.exists(BUILD_CEFPYTHON):
147+
print("[build_distirb.py] Delete directory: {dir}/"
148+
.format(dir=os.path.basename(BUILD_CEFPYTHON)))
149+
shutil.rmtree(BUILD_CEFPYTHON)
150+
# Delete cefpython_binary_*/ dirs
151+
delete_cefpython_binary_dir("32bit")
152+
delete_cefpython_binary_dir("64bit")
153+
154+
# Delete cef binaries and libraries dirs
128155
delete_cef_binaries_libraries_dir("32bit")
129156
delete_cef_binaries_libraries_dir("64bit")
130157

@@ -344,6 +371,47 @@ def zip_directory(path, base_path, archive):
344371
os.chdir(original_dir)
345372

346373

374+
def reduce_package_size_issue_262(arch):
375+
"""Linux only: libcef.so is huge (500 MB) in Chrome v54+. Issue #262."""
376+
print("[build_distrib.py] Reduce package size for {arch} (Issue #262)"
377+
.format(arch=arch))
378+
prebuilt_basename = get_cef_binaries_libraries_basename(
379+
get_postfix2_for_arch(arch))
380+
bin_dir = os.path.join(prebuilt_basename, "bin")
381+
382+
# Run `strip` command on `libcef.so`
383+
libcef_so = os.path.join(bin_dir, "libcef.so")
384+
print("[build_distrib.py] Strip {libcef_so}"
385+
.format(libcef_so=os.path.basename(libcef_so)))
386+
command = "strip {libcef_so}".format(libcef_so=libcef_so)
387+
pcode = subprocess.call(command)
388+
if pcode != 0:
389+
print("[build_distrib.py] ")
390+
sys.exit(1)
391+
392+
393+
def reduce_package_size_issue_321(arch):
394+
"""PyPI has file size limit and must reduce package size. Issue #321."""
395+
print("[build_distrib.py] Reduce package size for {arch} (Issue #321)"
396+
.format(arch=arch))
397+
prebuilt_basename = get_cef_binaries_libraries_basename(
398+
get_postfix2_for_arch(arch))
399+
bin_dir = os.path.join(prebuilt_basename, "bin")
400+
401+
# Delete sample applications to reduce package size
402+
sample_apps = ["cefclient", "cefsimple", "ceftests"]
403+
for sample_app in sample_apps:
404+
sample_app = os.path.join(bin_dir, sample_app + APP_EXT)
405+
# Not on all platforms sample apps may be available
406+
if os.path.exists(sample_app):
407+
print("[build_distrib.py] Delete {sample_app}"
408+
.format(sample_app=os.path.basename(sample_app)))
409+
if os.path.isdir(sample_app):
410+
shutil.rmtree(sample_app)
411+
else:
412+
os.remove(sample_app)
413+
414+
347415
def build_cefpython_modules(pythons):
348416
for python in pythons:
349417
print("[build_distrib.py] Build cefpython module for {python_name}"
@@ -417,9 +485,28 @@ def make_packages(python, arch):
417485
shutil.rmtree(setup_dir)
418486

419487

420-
def test_wheel_package_installation():
421-
# PYPI_POSTFIX2_ARCH
422-
pass # TODO
488+
def test_wheel_package_installation(pythons):
489+
uninstall_cefpython3_packages(pythons)
490+
for python in pythons:
491+
print("[build_distrib.py] Test wheel package installation for"
492+
" {python_name}".format(python_name=python["name"]))
493+
platform_tag = get_pypi_postfix2_for_arch(python["arch"])
494+
whl_pattern = (r"*-{platform_tag}.whl"
495+
.format(platform_tag=platform_tag))
496+
wheels = glob.glob(os.path.join(DISTRIB_DIR, whl_pattern))
497+
assert len(wheels) == 1, ("No wheels found in distrib dir for %s"
498+
% python["arch"])
499+
# Install wheel
500+
command = ("\"{python}\" -m pip install {wheel}"
501+
.format(python=python["executable"],
502+
wheel=os.path.basename(wheels[0])))
503+
if python["executable"].startswith("/usr/"):
504+
command = "sudo {command}".format(command=command)
505+
pcode = subprocess.call(command, cwd=DISTRIB_DIR)
506+
if pcode != 0:
507+
print("[build_distrib.py] Wheel package installation failed for"
508+
" {python_name}".format(python_name=python["name"]))
509+
sys.exit(1)
423510

424511

425512
def show_summary(pythons_32bit, pythons_64bit):
@@ -436,7 +523,7 @@ def show_summary(pythons_32bit, pythons_64bit):
436523
count=len(files)))
437524
for file_ in files:
438525
print(" {filename}".format(filename=os.path.basename(file_)))
439-
print("[build_distrib.py] Done. Distribution packages created.")
526+
print("[build_distrib.py] Everything OK. Distribution packages created.")
440527

441528

442529
if __name__ == "__main__":

‎tools/common.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
Darwin={"32bit": "mac32", "64bit": "mac64"},
5252
)
5353
PYPI_POSTFIX2_ARCH = dict(
54-
Windows={"32bit": "win32", "64bit": "win-amd64"},
54+
Windows={"32bit": "win32", "64bit": "win_amd64"},
5555
Linux={"32bit": "i686", "64bit": "x86_64"},
5656
Darwin={"64bit": "x86_64"},
5757
)
@@ -71,12 +71,15 @@
7171
MODULE_NAME = MODULE_NAME_TEMPLATE.format(pyversion=PYVERSION, ext=MODULE_EXT)
7272
MODULE_NAME_NOEXT = MODULE_NAME_TEMPLATE_NOEXT.format(pyversion=PYVERSION)
7373

74-
# Executable extension
74+
# App and Executable extensions
7575
if WINDOWS:
76+
APP_EXT = ".exe"
7677
EXECUTABLE_EXT = ".exe"
7778
elif MAC:
78-
EXECUTABLE_EXT = ".app"
79+
APP_EXT = ".app" # cefclient, cefsimple, ceftests
80+
EXECUTABLE_EXT = "" # subprocess
7981
else:
82+
APP_EXT = ""
8083
EXECUTABLE_EXT = ""
8184

8285
# Library extension
@@ -193,6 +196,10 @@ def get_postfix2_for_arch(arch):
193196
return OS_POSTFIX2_ARCH[SYSTEM][arch]
194197

195198

199+
def get_pypi_postfix2_for_arch(arch):
200+
return PYPI_POSTFIX2_ARCH[SYSTEM][arch]
201+
202+
196203
def _detect_cef_binaries_libraries_dir():
197204
"""Detect cef binary directory created by automate.py
198205
eg. build/cef55_3.2883.1553.g80bd606_win32/

0 commit comments

Comments
 (0)