Skip to content

Commit dc103fe

Browse files
wesmxhochy
authored andcommitted
ARROW-557: [Python] Add option to explicitly opt in to HDFS tests, do not implicitly skip
I have ``` $ py.test pyarrow/tests/test_hdfs.py ================================== test session starts ================================== platform linux2 -- Python 2.7.11, pytest-2.9.0, py-1.4.31, pluggy-0.3.1 rootdir: /home/wesm/code/arrow/python, inifile: collected 15 items pyarrow/tests/test_hdfs.py sssssssssssssss ``` But ``` $ py.test pyarrow/tests/test_hdfs.py --hdfs -v ================================== test session starts ================================== platform linux2 -- Python 2.7.11, pytest-2.9.0, py-1.4.31, pluggy-0.3.1 -- /home/wesm/anaconda3/envs/py27/bin/python cachedir: .cache rootdir: /home/wesm/code/arrow/python, inifile: collected 15 items pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_close PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_download_upload PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_file_context_manager PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_ls PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_mkdir PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_orphaned_file PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_read_multiple_parquet_files SKIPPED pyarrow/tests/test_hdfs.py::TestLibHdfs::test_hdfs_read_whole_file PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_close PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_download_upload PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_file_context_manager PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_ls PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_mkdir PASSED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_read_multiple_parquet_files SKIPPED pyarrow/tests/test_hdfs.py::TestLibHdfs3::test_hdfs_read_whole_file PASSED ``` The `py.test pyarrow --only-hdfs` option will run only the HDFS tests. Author: Wes McKinney <wes.mckinney@twosigma.com> Closes apache#353 from wesm/ARROW-557 and squashes the following commits: 52e03db [Wes McKinney] Add conftest.py file, hdfs group to opt in to HDFS tests with --hdfs
1 parent 16c9759 commit dc103fe

4 files changed

Lines changed: 69 additions & 14 deletions

File tree

LICENSE.txt

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -200,15 +200,3 @@
200200
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201201
See the License for the specific language governing permissions and
202202
limitations under the License.
203-
204-
--------------------------------------------------------------------------------
205-
206-
This product includes code from Apache Kudu.
207-
208-
* cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
209-
210-
Copyright: 2016 The Apache Software Foundation.
211-
Home page: https://kudu.apache.org/
212-
License: http://www.apache.org/licenses/LICENSE-2.0
213-
214-
--------------------------------------------------------------------------------

NOTICE.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ This product includes software from the CMake project
4242
This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
4343
* Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
4444

45+
This product includes software from the Ibis project (Apache 2.0)
46+
* Copyright (c) 2015 Cloudera, Inc.
47+
* https://github.com/cloudera/ibis
48+
4549
--------------------------------------------------------------------------------
4650

4751
This product includes code from Apache Kudu, which includes the following in

python/pyarrow/tests/conftest.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
from pytest import skip
19+
20+
21+
groups = ['hdfs']
22+
23+
24+
def pytest_configure(config):
25+
pass
26+
27+
28+
def pytest_addoption(parser):
29+
for group in groups:
30+
parser.addoption('--{0}'.format(group), action='store_true',
31+
default=False,
32+
help=('Enable the {0} test group'.format(group)))
33+
34+
for group in groups:
35+
parser.addoption('--only-{0}'.format(group), action='store_true',
36+
default=False,
37+
help=('Run only the {0} test group'.format(group)))
38+
39+
40+
def pytest_runtest_setup(item):
41+
only_set = False
42+
43+
for group in groups:
44+
only_flag = '--only-{0}'.format(group)
45+
flag = '--{0}'.format(group)
46+
47+
if item.config.getoption(only_flag):
48+
only_set = True
49+
elif getattr(item.obj, group, None):
50+
if not item.config.getoption(flag):
51+
skip('{0} NOT enabled'.format(flag))
52+
53+
if only_set:
54+
skip_item = True
55+
for group in groups:
56+
only_flag = '--only-{0}'.format(group)
57+
if (getattr(item.obj, group, False) and
58+
item.config.getoption(only_flag)):
59+
skip_item = False
60+
61+
if skip_item:
62+
skip('Only running some groups with only flags')

python/pyarrow/tests/test_hdfs.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ def hdfs_test_client(driver='libhdfs'):
4848
return HdfsClient(host, port, user, driver=driver)
4949

5050

51+
@pytest.mark.hdfs
5152
class HdfsTestCases(object):
5253

5354
def _make_test_file(self, hdfs, test_name, test_path, test_data):
@@ -190,7 +191,7 @@ class TestLibHdfs(HdfsTestCases, unittest.TestCase):
190191
@classmethod
191192
def check_driver(cls):
192193
if not io.have_libhdfs():
193-
pytest.skip('No libhdfs available on system')
194+
pytest.fail('No libhdfs available on system')
194195

195196
def test_hdfs_orphaned_file(self):
196197
hdfs = hdfs_test_client()
@@ -209,4 +210,4 @@ class TestLibHdfs3(HdfsTestCases, unittest.TestCase):
209210
@classmethod
210211
def check_driver(cls):
211212
if not io.have_libhdfs3():
212-
pytest.skip('No libhdfs3 available on system')
213+
pytest.fail('No libhdfs3 available on system')

0 commit comments

Comments
 (0)