Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,22 @@ generated using Sphinx. Documentation is also hosted on readthedocs.
:master: http://python-lz4.readthedocs.io/en/stable/
:development: http://python-lz4.readthedocs.io/en/latest/

Building on z/OS
================

The package can be built on z/OS UNIX System Services with the IBM Open
Enterprise SDK for Python and IBM Open XL C/C++ 2.2. The bundled LZ4 sources
are always used, and the extensions are compiled in ASCII mode to match the
Python interpreter::

$ export CC=ibm-clang64
$ export CXX=ibm-clang++64
$ export SETUPTOOLS_SCM_PRETEND_VERSION=<version> # if not a git checkout
$ python -m pip wheel . --no-deps --no-build-isolation -w dist

Use the ``python -m pip`` of the interpreter you want to install into, since the
wheel is tagged for that Python version. See ``docs/install.rst`` for details.

Homepage
========

Expand Down
15 changes: 15 additions & 0 deletions docs/install.rst
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,21 @@ They can be installed using ``pip``::

$ pip install -r requirements.txt

Building on z/OS USS
--------------------

The package can be built on z/OS UNIX System Services using the IBM Open
Enterprise SDK for Python and IBM Open XL C/C++ 2.2 (``ibm-clang64``). On z/OS
the bundled LZ4 sources are always used (system ``liblz4`` and ``pkg-config``
are ignored). ``setup.py`` adds ``-fzos-le-char-mode=ascii -m64`` unless a
character mode / ``-m64`` is already present in ``CC`` or ``CFLAGS``, so that
the extension matches the (ASCII) Python interpreter::

$ export CC=ibm-clang64
$ export CXX=ibm-clang++64 # required by the IBM-patched distutils
$ export SETUPTOOLS_SCM_PRETEND_VERSION=<version> # if not building from a git checkout
$ pip install .

Test suite
----------

Expand Down
33 changes: 28 additions & 5 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import new_compiler
import sys
import sysconfig

# z/OS USS (IBM Open XL C/C++, i.e. ibm-clang64). Python reports 'zos'.
is_zos = sys.platform == 'zos' or os.environ.get('PYLZ4_FORCE_ZOS', '') == '1'


# Note: if updating LZ4_REQUIRED_VERSION you need to update docs/install.rst as
Expand All @@ -14,6 +18,9 @@
liblz4_found = False

try:
if is_zos:
# The bundled lz4 sources are always used on z/OS.
raise ImportError
from pkgconfig import installed as pkgconfig_installed
from pkgconfig import parse as pkgconfig_parse
except ImportError:
Expand Down Expand Up @@ -68,7 +75,9 @@ def pkgconfig_installed_check(lib, required_version, default):
]

use_system_liblz4_env = os.environ.get("PYLZ4_USE_SYSTEM_LZ4", "True")
if use_system_liblz4_env.upper() in ("1", "TRUE"):
if is_zos:
use_system_liblz4 = False
elif use_system_liblz4_env.upper() in ("1", "TRUE"):
use_system_liblz4 = True
else:
use_system_liblz4 = False
Expand Down Expand Up @@ -112,6 +121,22 @@ def pkgconfig_installed_check(lib, required_version, default):
'/wd4711',
'/wd4820',
]
elif is_zos:
# IBM Open XL C/C++ (ibm-clang64). The extension must be compiled in the
# same character mode as the Python interpreter (ASCII for the IBM Open
# Enterprise SDK for Python), unless the user's flags already set one.
zos_flags = ' '.join([
os.environ.get('CC', ''),
os.environ.get('CFLAGS', ''),
sysconfig.get_config_var('CC') or '',
sysconfig.get_config_var('CFLAGS') or '',
])
extra = ['-O3', '-Wall', '-Wundef']
if '-fzos-le-char-mode' not in zos_flags:
extra.append('-fzos-le-char-mode=ascii')
if '-m64' not in zos_flags:
extra.append('-m64')
extension_kwargs['extra_compile_args'] = extra
elif compiler in ('unix', 'mingw32'):
if liblz4_found is True and use_system_liblz4 is True:
extension_kwargs = pkgconfig_parse('liblz4')
Expand Down Expand Up @@ -159,9 +184,8 @@ def pkgconfig_installed_check(lib, required_version, default):
# https://github.com/pytest-dev/pytest/issues/2957
tests_require = [
'pytest!=3.3.0',
'psutil',
'pytest-cov',
],
] + ([] if is_zos else ['psutil']), # psutil is not available on z/OS

# Only require pytest-runner if actually running the tests
needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv)
Expand All @@ -176,8 +200,7 @@ def pkgconfig_installed_check(lib, required_version, default):
python_requires=">=3.9",
setup_requires=[
'setuptools_scm',
'pkgconfig',
] + pytest_runner,
] + ([] if is_zos else ['pkgconfig']) + pytest_runner,
description="LZ4 Bindings for Python",
long_description=open('README.rst', 'r').read(),
author='Jonathan Underwood',
Expand Down
12 changes: 10 additions & 2 deletions tests/block/test_block_2.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
import pytest
import sys
import lz4.block
import psutil
import os

try:
import psutil
except ImportError:
psutil = None

_NO_PSUTIL = 'psutil is not installed; ' \
'cannot determine available memory'

# This test requires allocating a big lump of memory. In order to
# avoid a massive memory allocation during byte compilation, we have
# to declare a variable for the size of the buffer we're going to
Expand Down Expand Up @@ -32,9 +39,10 @@
reason='Py_ssize_t too small for this test'
)
@pytest.mark.skipif(
psutil.virtual_memory().available < _4GB,
psutil is not None and psutil.virtual_memory().available < _4GB,
reason='Insufficient system memory for this test'
)
@pytest.mark.skipif(psutil is None, reason=_NO_PSUTIL)
def test_huge():
try:
huge = b'\0' * _4GB
Expand Down
5 changes: 5 additions & 0 deletions tests/frame/test_frame_6.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@
import threading
import lz4.frame as lz4frame

pytestmark = pytest.mark.skipif(
not threading._HAVE_THREAD_NATIVE_ID,
reason='needs native_id'
)

test_data = [
b'',
(128 * (32 * os.urandom(32))),
Expand Down
6 changes: 6 additions & 0 deletions tests/frame/test_frame_8.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import threading
import pytest
import lz4.frame as lz4frame

pytestmark = pytest.mark.skipif(
not threading._HAVE_THREAD_NATIVE_ID,
reason='needs native_id'
)


def test_lz4frame_open_write_read_text_iter(tmp_path):
data = u'This is a test string'
Expand Down
8 changes: 8 additions & 0 deletions tests/frame/test_frame_9.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,13 @@
import lz4.frame
import pytest

needs_native_id = pytest.mark.skipif(
not threading._HAVE_THREAD_NATIVE_ID,
reason='needs native_id'
)


@needs_native_id
def test_issue_172_1(tmp_path):
"""Test reproducer for issue 172

Expand All @@ -27,6 +33,7 @@ def test_issue_172_1(tmp_path):
assert len(data) == 10


@needs_native_id
def test_issue_172_2(tmp_path):
input_data = 9 * os.urandom(1024)
thread_id = threading.get_native_id()
Expand All @@ -38,6 +45,7 @@ def test_issue_172_2(tmp_path):
assert len(data) == 10


@needs_native_id
def test_issue_172_3(tmp_path):
input_data = 9 * os.urandom(1024)
thread_id = threading.get_native_id()
Expand Down
22 changes: 16 additions & 6 deletions tests/stream/test_stream_2.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
import pytest
import sys
import lz4.stream
import psutil
import os

try:
import psutil
except ImportError:
psutil = None

_NO_PSUTIL = 'psutil is not installed; ' \
'cannot determine available memory'


# This test requires allocating a big lump of memory. In order to
# avoid a massive memory allocation during byte compilation, we have
Expand All @@ -20,8 +27,8 @@
# check for the TRAVIS environment variable being set. This is quite
# fragile.

if os.environ.get('TRAVIS') is not None or sys.maxsize < _4GB or \
psutil.virtual_memory().available < _4GB:
if psutil is None or os.environ.get('TRAVIS') is not None or \
sys.maxsize < _4GB or psutil.virtual_memory().available < _4GB:
huge = None
else:
try:
Expand All @@ -39,9 +46,10 @@
reason='Py_ssize_t too small for this test'
)
@pytest.mark.skipif(
psutil.virtual_memory().available < _4GB or huge is None,
psutil is not None and (psutil.virtual_memory().available < _4GB or huge is None),
reason='Insufficient system memory for this test'
)
@pytest.mark.skipif(psutil is None, reason=_NO_PSUTIL)
def test_huge_1():
data = b''
kwargs = {
Expand Down Expand Up @@ -78,9 +86,10 @@ def test_huge_1():
reason='Py_ssize_t too small for this test'
)
@pytest.mark.skipif(
psutil.virtual_memory().available < _4GB or huge is None,
psutil is not None and (psutil.virtual_memory().available < _4GB or huge is None),
reason='Insufficient system memory for this test'
)
@pytest.mark.skipif(psutil is None, reason=_NO_PSUTIL)
def test_huge_2():
data = huge
kwargs = {
Expand Down Expand Up @@ -118,9 +127,10 @@ def test_huge_2():
reason='Py_ssize_t too small for this test'
)
@pytest.mark.skipif(
psutil.virtual_memory().available < _4GB or huge is None,
psutil is not None and (psutil.virtual_memory().available < _4GB or huge is None),
reason='Insufficient system memory for this test'
)
@pytest.mark.skipif(psutil is None, reason=_NO_PSUTIL)
def test_huge_3():
data = huge
kwargs = {
Expand Down