Skip to content

Commit c472898

Browse files
cztomczakcztomczak
committed
Fix tools on Linux. Add 32-bit support on Linux (cztomczak#327).
1 parent d0844b1 commit c472898

File tree

13 files changed

+147
-83
lines changed

13 files changed

+147
-83
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ scraping or as a web crawler, or other kind of internet bots.
6060
- Downloads are available on [GitHub Releases](../../releases) pages
6161
- Windows support: 32-bit and 64-bit, Python 2.7 / 3.4 / 3.5 / 3.6
6262
(requirements: Windows 7+)
63-
- Linux support: 64-bit, Python 2.7 / 3.4 / 3.5 / 3.6
63+
- Linux support: 32-bit and 64-bit, Python 2.7 / 3.4 / 3.5 / 3.6
6464
(requirements: Debian 7+ / Ubuntu 12.04+)
6565
- Mac support: 64-bit, Python 2.7 / 3.4 / 3.5 / 3.6
6666
(requirements: MacOS 10.9+)

‎docs/Build-instructions.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,10 +184,7 @@ requirements common for all platforms.
184184

185185
### Linux
186186

187-
* Install packages: `sudo apt-get install python-dev cmake g++ libgtk2.0-dev`
188-
* If using prebuilt binaries from Spotify automated builds and want to
189-
build cefclient/cefsimple you need to install these packages:
190-
`sudo apt-get install libgtkglext1-dev`
187+
* Install packages: `sudo apt-get install python-dev cmake g++ libgtk2.0-dev libgtkglext1-dev`
191188
* If building CEF from sources:
192189
* Official binaries are built on Ubuntu 14.04 (cmake 2.8.12, g++ 4.8.4)
193190
* Download [ninja](http://martine.github.io/ninja/) 1.7.1 or later
@@ -209,6 +206,8 @@ requirements common for all platforms.
209206
* To perform a 32-bit Linux build on a 64-bit Linux system see
210207
Linux configuration in upstream cef/AutomatedBuildSetup.md. See also
211208
[cef/#1804](https://bitbucket.org/chromiumembedded/cef/issues/1804).
209+
* Sometimes it is also required to install these packages (eg. chroot):
210+
`sudo apt-get install libnss3 libnspr4 libxss1 libgconf-2-4`
212211

213212

214213
### Mac

‎examples/Examples-README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ maintained:
2121
and [PySide](https://wiki.qt.io/PySide) libraries (Qt 4)
2222
- [tkinter_.py](tkinter_.py): example for [Tkinter]
2323
(https://wiki.python.org/moin/TkInter). Currently broken on Mac.
24-
- [wxpython.py](wxpython.py): example for wxPython
24+
- [wxpython.py](wxpython.py): example for [wxPython](https://wxpython.org/)
2525

2626
If there are any issues in examples read top comments in sources
2727
to see whether this is a known issue with available workarounds.

‎src/client_handler/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ OBJ = $(filter %.o, $(SRC:.cpp=.o) $(SRC:.mm=.o))
3030

3131
OUT = libclient_handler.a
3232

33-
INC = -I./../ -I./../common/ -I/usr/include/python2.7 \
33+
INC = -I./../ -I./../common/ -I$(PYTHON_INCLUDE) \
3434
-I/usr/include/gtk-2.0 \
3535
-I/usr/include/gtk-unix-print-2.0 \
3636
-I/usr/include/glib-2.0 \

‎src/subprocess/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# -Werror - treat warnings as errors
66

77
UNAME_S = $(shell uname -s)
8-
INC = -I./../ -I./../common/ -I/usr/include/python2.7 \
8+
INC = -I./../ -I./../common/ -I$(PYTHON_INCLUDE) \
99
-I/usr/include/gtk-2.0 \
1010
-I/usr/include/gtk-unix-print-2.0 \
1111
-I/usr/include/glib-2.0 \

‎src/subprocess/Makefile-libcefpythonapp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ OBJ = $(filter %.o, $(SRC:.cpp=.o) $(SRC:.mm=.o))
3030

3131
OUT = libcefpythonapp.a
3232

33-
INC = -I./../ -I./../common/ -I/usr/include/python2.7 \
33+
INC = -I./../ -I./../common/ -I$(PYTHON_INCLUDE) \
3434
-I/usr/include/gtk-2.0 \
3535
-I/usr/include/gtk-unix-print-2.0 \
3636
-I/usr/include/glib-2.0 \

‎tools/automate.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,6 @@ def main():
126126
" version of python.")
127127
sys.exit(1)
128128
build_cef()
129-
# Build cefclient, cefsimple, ceftests, libcef_dll_wrapper
130-
build_cef_projects()
131-
create_prebuilt_binaries()
132129
elif Options.prebuilt_cef:
133130
prebuilt_cef()
134131

@@ -209,6 +206,11 @@ def setup_options(docopt_args):
209206
def build_cef():
210207
"""Build CEF from sources."""
211208

209+
if ARCH32:
210+
print("[automate.py] INFO: building CEF 32-bit from sources is"
211+
" supported only with cross-compiling on 64-bit OS.")
212+
sys.exit(1)
213+
212214
# cef/ repo
213215
create_cef_directories()
214216

@@ -221,6 +223,18 @@ def build_cef():
221223
print("[automate.py] Binary distrib created in %s"
222224
% Options.binary_distrib)
223225

226+
if Options.x86:
227+
print("[automate.py] INFO: to build CEF projects and create prebuilt"
228+
" binaries you have to use 32-bit chroot. Copy the binary"
229+
" distrib's cef_binary_*/ directory (path displayed above) to"
230+
" cefpython's build/ directory. Then run automate.py"
231+
" --prebuilt-cef using 32-bit chroot.")
232+
sys.exit(0)
233+
else:
234+
# Build cefclient, cefsimple, ceftests, libcef_dll_wrapper
235+
build_cef_projects()
236+
create_prebuilt_binaries()
237+
224238

225239
def prebuilt_cef():
226240
"""Use prebuilt binaries."""
@@ -884,7 +898,10 @@ def run_command(command, working_dir, env=None):
884898
args = command
885899
if not env:
886900
env = getenv()
887-
return subprocess.check_call(args, cwd=working_dir, env=env, shell=True)
901+
# When passing list of args shell cannot be True on eg. Linux, read
902+
# notes in build.py
903+
shell=(platform.system() == "Windows")
904+
return subprocess.check_call(args, cwd=working_dir, env=env, shell=shell)
888905

889906

890907
def run_git(command_line, working_dir):

‎tools/build.py‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -30,20 +30,23 @@
3030

3131
# --rebuild-cpp Force rebuild of .vcproj C++ projects (DISABLED)
3232

33-
# How to debug on Linux:
33+
# NOTE: When passing string command to subprocess functions you must
34+
# always use shell=True, otherwise on Linux error is thrown:
35+
# "No such file or directory". Always pass string commands to
36+
# subprocess functions with shell=True. If you pass a list of
37+
# arguments instead, then on Linux a "Segmentation fault" error
38+
# message is not shown. When passing a list of args to subprocess
39+
# function then you can't pass shell=True on Linux. If you pass
40+
# then it will execute args[0] and ignore others args.
41+
42+
# How to debug on Linux (OLD unsupported).
3443
# 1. Install "python-dbg" package
3544
# 2. Install "python-wxgtk2.8-dbg" package
3645
# 3. Run "python compile.py debug"
3746
# 4. In cygdb type "cy run"
3847
# 5. To display debug backtrace type "cy bt"
3948
# 6. More commands: http://docs.cython.org/src/userguide/debugging.html
4049

41-
# This will not show "Segmentation fault" error message:
42-
# > subprocess.call(["python", "./wxpython.py"])
43-
# You need to call it with command as string and shell=True
44-
# for this kind of error message to be shown:
45-
# > subprocess.call("python wxpython.py", shell=True)
46-
4750
from common import *
4851
import sys
4952
import os
@@ -210,8 +213,7 @@ def setup_environ():
210213
if WINDOWS:
211214
if "INCLUDE" not in os.environ:
212215
os.environ["INCLUDE"] = ""
213-
os.environ["INCLUDE"] += os.pathsep + os.path.join(get_python_path(),
214-
"include")
216+
os.environ["INCLUDE"] += os.pathsep + get_python_include_path()
215217
print("[build.py] environ INCLUDE: {include}"
216218
.format(include=os.environ["INCLUDE"]))
217219

@@ -223,24 +225,26 @@ def setup_environ():
223225
.format(lib=os.environ["AdditionalLibraryDirectories"]))
224226

225227
if LINUX or MAC:
226-
# Used in makefiles
228+
# Env variables for makefiles
229+
os.environ["PYTHON_INCLUDE"] = get_python_include_path()
230+
print("[build.py] PYTHON_INCLUDE: {python_include}"
231+
.format(python_include=os.environ["PYTHON_INCLUDE"]))
232+
227233
os.environ["CEF_CCFLAGS"] = "-std=gnu++11 -DNDEBUG -Wall -Werror"
228234
if FAST_FLAG:
229235
os.environ["CEF_CCFLAGS"] += " -O0"
230236
else:
231237
os.environ["CEF_CCFLAGS"] += " -O3"
232238
os.environ["CEF_LINK_FLAGS"] = ""
233239

240+
os.environ["CEF_BIN"] = os.path.join(CEF_BINARIES_LIBRARIES, "bin")
241+
os.environ["CEF_LIB"] = os.path.join(CEF_BINARIES_LIBRARIES, "lib")
242+
234243
if LINUX:
235244
# TODO: Set CEF_CCFLAGS and CEF_LINK_FLAGS according to what is
236245
# in upstream cefclient, see cef/cmake/cef_variables.cmake.
237246
pass
238247

239-
# Mac env variables for makefiles
240-
if MAC or LINUX:
241-
os.environ["CEF_BIN"] = os.path.join(CEF_BINARIES_LIBRARIES, "bin")
242-
os.environ["CEF_LIB"] = os.path.join(CEF_BINARIES_LIBRARIES, "lib")
243-
244248
# Mac compiler options
245249
if MAC:
246250
os.environ["PATH"] = "/usr/local/bin:"+os.environ["PATH"]
@@ -282,11 +286,6 @@ def setup_environ():
282286
)
283287

284288

285-
def get_python_path():
286-
"""Get Python path."""
287-
return os.path.dirname(sys.executable)
288-
289-
290289
def fix_cefpython_api_header_file():
291290
"""This function does two things: 1) Disable warnings in cefpython
292291
API header file and 2) Make a copy named cefpython_pyXX_fixed.h,
@@ -449,6 +448,7 @@ def clean_cpp_projects_unix():
449448
delete_files_by_pattern("{0}/*.o".format(SUBPROCESS_DIR))
450449
delete_files_by_pattern("{0}/*.a".format(SUBPROCESS_DIR))
451450
delete_files_by_pattern("{0}/subprocess".format(SUBPROCESS_DIR))
451+
delete_files_by_pattern("{0}/main_message_loop/*.o".format(SUBPROCESS_DIR))
452452

453453
delete_files_by_pattern("{0}/*.o".format(CPP_UTILS_DIR))
454454
delete_files_by_pattern("{0}/*.a".format(CPP_UTILS_DIR))

‎tools/build_cpp_projects.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,6 @@ def smart_compile(compiler, macros, extra_args, sources, output_dir):
247247
# all put in the same output_directory. Otherwise distutils
248248
# will create lots of subdirs in output_directory.
249249
macros = macros_as_tuples(macros)
250-
python_include = os.path.join(os.path.dirname(sys.executable),
251-
"include")
252250
common_dir = os.path.join(SRC_DIR, "common")
253251
original_dir = os.getcwd()
254252
for source_file in sources:
@@ -261,7 +259,7 @@ def smart_compile(compiler, macros, extra_args, sources, output_dir):
261259
# TODO include dirs for Linux/Mac
262260
include_dirs=[SRC_DIR,
263261
common_dir,
264-
python_include],
262+
get_python_include_path()],
265263
# TODO compiler flags for Linux/Mac
266264
extra_preargs=None,
267265
extra_postargs=extra_args)

‎tools/build_distrib.py‎

Lines changed: 27 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
import re
6161
import shutil
6262
import subprocess
63-
import tarfile
63+
# import tarfile # Currently using zip on all platforms
6464
import zipfile
6565

6666
# Command line args
@@ -117,13 +117,13 @@ def main():
117117
pack_prebuilt_cef("32bit")
118118
if LINUX:
119119
reduce_package_size_issue_262("32bit")
120-
reduce_package_size_issue_321("32bit")
121-
if pythons_64bit is not None:
120+
remove_unnecessary_package_files("32bit")
121+
if pythons_64bit:
122122
run_automate_prebuilt_cef(pythons_64bit[0])
123123
pack_prebuilt_cef("64bit")
124124
if LINUX:
125125
reduce_package_size_issue_262("64bit")
126-
reduce_package_size_issue_321("64bit")
126+
remove_unnecessary_package_files("64bit")
127127
if not NO_REBUILD:
128128
build_cefpython_modules(pythons_32bit + pythons_64bit)
129129
if pythons_32bit:
@@ -264,19 +264,29 @@ def search_for_pythons(search_arch):
264264

265265

266266
def check_pythons(pythons_32bit, pythons_64bit):
267+
check_32bit = True
268+
check_64bit = True
269+
if MAC:
270+
check_32bit = False
271+
elif LINUX:
272+
if pythons_64bit:
273+
check_32bit = False
274+
elif pythons_32bit:
275+
check_64bit = False
276+
267277
pp = pprint.PrettyPrinter(indent=4)
268278
if pythons_32bit:
269279
print("[build_distrib.py] Pythons 32-bit found:")
270280
pp.pprint(pythons_32bit)
271-
if WINDOWS and len(pythons_32bit) != len(SUPPORTED_PYTHON_VERSIONS):
281+
if check_32bit and len(pythons_32bit) != len(SUPPORTED_PYTHON_VERSIONS):
272282
print("[build_distrib.py] ERROR: Couldn't find all supported"
273283
" python 32-bit installations. Found: {found}."
274284
.format(found=len(pythons_32bit)))
275285
sys.exit(1)
276286
if pythons_64bit:
277287
print("[build_distrib.py] Pythons 64-bit found:")
278288
pp.pprint(pythons_64bit)
279-
if len(pythons_64bit) != len(SUPPORTED_PYTHON_VERSIONS):
289+
if check_64bit and len(pythons_64bit) != len(SUPPORTED_PYTHON_VERSIONS):
280290
print("[build_distrib.py] ERROR: Couldn't find all supported"
281291
" python 64-bit installations. Found: {found}."
282292
.format(found=len(pythons_64bit)))
@@ -367,15 +377,17 @@ def pack_prebuilt_cef(arch):
367377
def pack_directory(path, base_path):
368378
if path.endswith(os.path.sep):
369379
path = path[:-1]
370-
ext = ".zip" if WINDOWS or MAC else ".tar.gz"
380+
# ext = ".zip" if WINDOWS or MAC else ".tar.gz"
381+
ext = ".zip"
371382
archive = path + ext
372383
if os.path.exists(archive):
373384
os.remove(archive)
374385
if WINDOWS or MAC:
375386
zip_directory(path, base_path=base_path, archive=archive)
376387
else:
377-
with tarfile.open(archive, "w:gz") as tar:
378-
tar.add(path, arcname=os.path.basename(path))
388+
zip_directory(path, base_path=base_path, archive=archive)
389+
# with tarfile.open(archive, "w:gz") as tar:
390+
# tar.add(path, arcname=os.path.basename(path))
379391
assert os.path.isfile(archive), archive
380392
return archive
381393

@@ -408,43 +420,19 @@ def reduce_package_size_issue_262(arch):
408420
.format(libcef_so=os.path.basename(libcef_so)))
409421
command = "strip {libcef_so}".format(libcef_so=libcef_so)
410422
pcode = subprocess.call(command, shell=True)
411-
assert pcode, "strip command failed"
423+
assert pcode == 0, "strip command failed"
412424

413425

414-
def reduce_package_size_issue_321(arch):
415-
"""PyPI has file size limit and must reduce package size. Issue #321."""
426+
def remove_unnecessary_package_files(arch):
427+
"""Do not ship sample applications (cefclient etc) with the package.
428+
They increase size and also are an additional unnecessary factor
429+
when dealing with false-positives in Anti-Virus software."""
416430
print("[build_distrib.py] Reduce package size for {arch} (Issue #321)"
417431
.format(arch=arch))
418432
prebuilt_basename = get_cef_binaries_libraries_basename(
419433
get_os_postfix2_for_arch(arch))
420434
bin_dir = os.path.join(prebuilt_basename, "bin")
421-
422-
# Delete sample applications to reduce package size
423-
sample_apps = ["cefclient", "cefsimple", "ceftests"]
424-
for sample_app_name in sample_apps:
425-
sample_app = os.path.join(bin_dir, sample_app_name + APP_EXT)
426-
# Not on all platforms sample apps may be available
427-
if os.path.exists(sample_app):
428-
print("[build_distrib.py] Delete {sample_app}"
429-
.format(sample_app=os.path.basename(sample_app)))
430-
if os.path.isdir(sample_app):
431-
shutil.rmtree(sample_app)
432-
else:
433-
os.remove(sample_app)
434-
# Also delete subdirs eg. cefclient_files/, ceftests_files/
435-
files_subdir = os.path.join(bin_dir, sample_app_name + "_files")
436-
if os.path.isdir(files_subdir):
437-
print("[build_distrib.py] Delete directory: {dir}/"
438-
.format(dir=os.path.basename(files_subdir)))
439-
shutil.rmtree(files_subdir)
440-
441-
# Strip symbols from cefpython .so modules to reduce size
442-
modules = glob.glob(os.path.join(CEFPYTHON_BINARY, "*.so"))
443-
for module in modules:
444-
print("[build_distrib.py] strip {module}"
445-
.format(module=os.path.basename(module)))
446-
command = "strip {module}".format(module=module)
447-
assert os.system(command) == 0, "strip command failed"
435+
delete_cef_sample_apps(caller_script=__file__, bin_dir=bin_dir)
448436

449437

450438
def build_cefpython_modules(pythons):

0 commit comments

Comments
 (0)