ARROW-4260: [Python] NumPy buffer protocol failure

kszucs · kszucs · commit 09d34964996a · 2019-01-15T17:02:26.000+01:00
The following error only occurred when both numpy=1.16 and pickle5 was installed. It's a numpy issue numpy/numpy#12745, so skipping the test until it's resolved. ``` ====================================================== FAILURES ======================================================= __________________________________________ test_serialize_deserialize_pandas __________________________________________ ValueError: cannot include dtype 'M' in a buffer The above exception was the direct cause of the following exception: def test_serialize_deserialize_pandas(): # ARROW-1784, serialize and deserialize DataFrame by decomposing # BlockManager df = _fully_loaded_dataframe_example() > _check_serialize_components_roundtrip(df) pyarrow/tests/test_convert_pandas.py:2278: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ pyarrow/tests/test_convert_pandas.py:2268: in _check_serialize_components_roundtrip components = ctx.serialize(df).to_components() pyarrow/serialization.pxi:198: in pyarrow.lib.SerializationContext.serialize return serialize(obj, context=self) pyarrow/serialization.pxi:345: in pyarrow.lib.serialize check_status(SerializeObject(context, wrapped_value, &serialized.data)) pyarrow/serialization.pxi:153: in pyarrow.lib.SerializationContext._serialize_callback serialized_obj = {"data": self.custom_serializers[type_id](obj)} _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ x = MultiIndex(levels=[[2000-01-01 00:00:00, 2000-01-02 00:00:00, 2000-01-03 00:00:00, 2000-01-04 00:00:00, 2000-01-05 00:00:00], ['bar', 'foo']], labels=[[0, 0, 1, 1, 2, 2, 3, 3, 4, 4], [1, 0, 1, 0, 1, 0, 1, 0, 1, 0]]) def _pickle_to_buffer(x): > pickled = builtin_pickle.dumps(x, protocol=builtin_pickle.HIGHEST_PROTOCOL) E SystemError: <built-in function __import__> returned a result with an error set ``` Author: Krisztián Szűcs <szucs.krisztian@gmail.com> Closes apache#3403 from kszucs/travis_numpy_version and squashes the following commits: e3162a8 <Krisztián Szűcs> pin to minor 77e9b64 <Krisztián Szűcs> reason 301226d <Krisztián Szűcs> skip serialize test 39a7161 <Krisztián Szűcs> appveyor ae1f16d <Krisztián Szűcs> pin numpy to 1.14
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
@@ -95,11 +95,12 @@ if "%JOB%" == "Build_Debug" (
   exit /B 0
 )
 
-conda create -n arrow -q -y ^
+conda create -n arrow -q -y -c conda-forge ^
+      --file=ci\conda_env_python.yml ^
       python=%PYTHON% ^
-      six pytest setuptools numpy pandas cython hypothesis ^
-      thrift-cpp=0.11.0 boost-cpp ^
-      -c conda-forge
+      numpy=1.14 ^
+      thrift-cpp=0.11 ^
+      boost-cpp
 
 call activate arrow
 
@@ -109,9 +110,9 @@ set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
 
 if "%JOB%" == "Toolchain" (
   @rem Install pre-built "toolchain" packages for faster builds
-  conda install -q -y --file=ci\conda_env_cpp.yml ^
-        python=%PYTHON% ^
-        -c conda-forge
+  conda install -q -y -c conda-forge ^
+        --file=ci\conda_env_cpp.yml ^
+        python=%PYTHON%
 
   set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 )
diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat
@@ -72,7 +72,7 @@ popd
 
 pushd python
 
-pip install pickle5
+pip install -r requirements.txt pickle5
 
 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS%
 set PYARROW_CMAKE_GENERATOR=%GENERATOR%
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
@@ -50,7 +50,7 @@ conda create -y -q -p $CONDA_ENV_DIR \
       nomkl \
       cmake \
       pip \
-      numpy=1.13.1 \
+      numpy=1.14 \
       python=${PYTHON_VERSION} \
       ${CONDA_JVM_DEPS}
 
@@ -124,7 +124,7 @@ $ARROW_CPP_BUILD_DIR/$ARROW_BUILD_TYPE/arrow-python-test
 pushd $ARROW_PYTHON_DIR
 
 # Other stuff pip install
-pip install -q -r requirements.txt
+pip install -r requirements.txt
 
 if [ "$PYTHON_VERSION" == "3.6" ]; then
     pip install -q pickle5
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
@@ -23,6 +23,7 @@
 
 from collections import OrderedDict
 from datetime import date, datetime, time, timedelta
+from distutils.version import LooseVersion
 
 import hypothesis as h
 import hypothesis.extra.pytz as tzst
@@ -2224,8 +2225,6 @@ def test_safe_unsafe_casts(self):
 
 
 def _fully_loaded_dataframe_example():
-    from distutils.version import LooseVersion
-
     index = pd.MultiIndex.from_arrays([
         pd.date_range('2000-01-01', periods=5).repeat(2),
         np.tile(np.array(['foo', 'bar'], dtype=object), 5)
@@ -2271,6 +2270,8 @@ def _check_serialize_components_roundtrip(df):
     tm.assert_frame_equal(df, deserialized)
 
 
+@pytest.mark.skipif(LooseVersion(np.__version__) >= '0.16',
+                    reason='Until numpy/numpy#12745 is resolved')
 def test_serialize_deserialize_pandas():
     # ARROW-1784, serialize and deserialize DataFrame by decomposing
     # BlockManager
