# 'a', 'b'... 'z', then 'aa', 'ab'...

letters = string.ascii_lowercase

yield from letters

for first in letters:

for second in letters:

arr = np.random.RandomState(42).randint(0, 1000, size=(num_cols, num_rows))

csv = io.StringIO()

col_names = list(itertools.islice(generate_col_names(), num_cols))

if write_names:

csv.write(",".join(col_names))

csv.write(linesep)

for row in arr.T:

csv.write(",".join(map(str, row)))

csv.write(linesep)

csv = csv.getvalue().encode()

columns = [pa.array(a, type=pa.int64()) for a in arr]

expected = pa.Table.from_arrays(columns, col_names)

csv = io.StringIO()

csv.write(",".join(column_names))

csv.write("\n")

"""

opts = cls()

for name, values in attr_values.items():

assert getattr(opts, name) == values[0], \

"incorrect default value for " + name

for v in values:

setattr(opts, name, v)

assert getattr(opts, name) == v, "failed setting value"

with pytest.raises(AttributeError):

opts.zzz_non_existent = True

# Check constructor named arguments

non_defaults = {name: values[1] for name, values in attr_values.items()}

opts = cls(**non_defaults)

for name, value in non_defaults.items():

opts = cls(**attr_values)

new_opts = pickle.loads(pickle.dumps(opts,

protocol=pickle.HIGHEST_PROTOCOL))

for name, value in attr_values.items():

def __init__(self, result):

self.result = result

self.rows = []

def __call__(self, row):

self.rows.append(row)

return self.result

def __eq__(self, other):

return (isinstance(other, InvalidRowHandler) and

other.result == self.result)

def __ne__(self, other):

return (not isinstance(other, InvalidRowHandler) or

cls = ReadOptions

opts = cls()

check_options_class(cls, use_threads=[True, False],

skip_rows=[0, 3],

column_names=[[], ["ab", "cd"]],

autogenerate_column_names=[False, True],

encoding=['utf8', 'utf16'],

skip_rows_after_names=[0, 27])

check_options_class_pickling(cls, use_threads=True,

skip_rows=3,

column_names=["ab", "cd"],

autogenerate_column_names=False,

encoding='utf16',

skip_rows_after_names=27)

assert opts.block_size > 0

opts.block_size = 12345

assert opts.block_size == 12345

opts = cls(block_size=1234)

assert opts.block_size == 1234

opts.validate()

match = "ReadOptions: block_size must be at least 1: 0"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.block_size = 0

opts.validate()

match = "ReadOptions: skip_rows cannot be negative: -1"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.skip_rows = -1

opts.validate()

match = "ReadOptions: skip_rows_after_names cannot be negative: -1"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.skip_rows_after_names = -1

opts.validate()

match = "ReadOptions: autogenerate_column_names cannot be true when" \

" column_names are provided"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.autogenerate_column_names = True

opts.column_names = ('a', 'b')

cls = ParseOptions

skip_handler = InvalidRowHandler('skip')

check_options_class(cls, delimiter=[',', 'x'],

escape_char=[False, 'y'],

quote_char=['"', 'z', False],

double_quote=[True, False],

newlines_in_values=[False, True],

ignore_empty_lines=[True, False],

invalid_row_handler=[None, skip_handler])

check_options_class_pickling(cls, delimiter='x',

escape_char='y',

quote_char=False,

double_quote=False,

newlines_in_values=True,

ignore_empty_lines=False,

invalid_row_handler=skip_handler)

cls().validate()

opts = cls()

opts.delimiter = "\t"

opts.validate()

match = "ParseOptions: delimiter cannot be \\\\r or \\\\n"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.delimiter = "\n"

opts.validate()

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.delimiter = "\r"

opts.validate()

match = "ParseOptions: quote_char cannot be \\\\r or \\\\n"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.quote_char = "\n"

opts.validate()

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.quote_char = "\r"

opts.validate()

match = "ParseOptions: escape_char cannot be \\\\r or \\\\n"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.escape_char = "\n"

opts.validate()

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.escape_char = "\r"

cls = ConvertOptions

opts = cls()

check_options_class(

cls, check_utf8=[True, False],

strings_can_be_null=[False, True],

quoted_strings_can_be_null=[True, False],

decimal_point=['.', ','],

include_columns=[[], ['def', 'abc']],

include_missing_columns=[False, True],

auto_dict_encode=[False, True],

timestamp_parsers=[[], [ISO8601, '%y-%m']])

check_options_class_pickling(

cls, check_utf8=False,

strings_can_be_null=True,

quoted_strings_can_be_null=False,

decimal_point=',',

include_columns=['def', 'abc'],

include_missing_columns=False,

auto_dict_encode=True,

timestamp_parsers=[ISO8601, '%y-%m'])

with pytest.raises(ValueError):

opts.decimal_point = '..'

assert opts.auto_dict_max_cardinality > 0

opts.auto_dict_max_cardinality = 99999

assert opts.auto_dict_max_cardinality == 99999

assert opts.column_types == {}

# Pass column_types as mapping

opts.column_types = {'b': pa.int16(), 'c': pa.float32()}

assert opts.column_types == {'b': pa.int16(), 'c': pa.float32()}

opts.column_types = {'v': 'int16', 'w': 'null'}

assert opts.column_types == {'v': pa.int16(), 'w': pa.null()}

# Pass column_types as schema

schema = pa.schema([('a', pa.int32()), ('b', pa.string())])

opts.column_types = schema

assert opts.column_types == {'a': pa.int32(), 'b': pa.string()}

# Pass column_types as sequence

opts.column_types = [('x', pa.binary())]

assert opts.column_types == {'x': pa.binary()}

with pytest.raises(TypeError, match='DataType expected'):

opts.column_types = {'a': None}

with pytest.raises(TypeError):

opts.column_types = 0

assert isinstance(opts.null_values, list)

assert '' in opts.null_values

assert 'N/A' in opts.null_values

opts.null_values = ['xxx', 'yyy']

assert opts.null_values == ['xxx', 'yyy']

assert isinstance(opts.true_values, list)

opts.true_values = ['xxx', 'yyy']

assert opts.true_values == ['xxx', 'yyy']

assert isinstance(opts.false_values, list)

opts.false_values = ['xxx', 'yyy']

assert opts.false_values == ['xxx', 'yyy']

assert opts.timestamp_parsers == []

opts.timestamp_parsers = [ISO8601]

assert opts.timestamp_parsers == [ISO8601]

opts = cls(column_types={'a': pa.null()},

null_values=['N', 'nn'], true_values=['T', 'tt'],

false_values=['F', 'ff'], auto_dict_max_cardinality=999,

timestamp_parsers=[ISO8601, '%Y-%m-%d'])

assert opts.column_types == {'a': pa.null()}

assert opts.null_values == ['N', 'nn']

assert opts.false_values == ['F', 'ff']

assert opts.true_values == ['T', 'tt']

assert opts.auto_dict_max_cardinality == 999

cls = WriteOptions

opts = cls()

check_options_class(

cls, include_header=[True, False], delimiter=[',', '\t', '|'],

quoting_style=['needed', 'none', 'all_valid'])

assert opts.batch_size > 0

opts.batch_size = 12345

assert opts.batch_size == 12345

opts = cls(batch_size=9876)

assert opts.batch_size == 9876

opts.validate()

match = "WriteOptions: batch_size must be at least 1: 0"

with pytest.raises(pa.ArrowInvalid, match=match):

opts = cls()

opts.batch_size = 0

"""Common tests which are shared by streaming and non streaming readers"""

@abc.abstractmethod

def read_bytes(self, b, **kwargs):

"""

raise NotImplementedError

@property

@abc.abstractmethod

def use_threads(self):

"""Whether this test is multi-threaded"""

raise NotImplementedError

@staticmethod

def check_names(table, names):

assert table.num_columns == len(names)

assert table.column_names == names

def test_header_skip_rows(self):

rows = b"ab,cd\nef,gh\nij,kl\nmn,op\n"

opts = ReadOptions()

opts.skip_rows = 1

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["ef", "gh"])

assert table.to_pydict() == {

"ef": ["ij", "mn"],

"gh": ["kl", "op"],

}

opts.skip_rows = 3

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["mn", "op"])

assert table.to_pydict() == {

"mn": [],

"op": [],

}

opts.skip_rows = 4

with pytest.raises(pa.ArrowInvalid):

# Not enough rows

table = self.read_bytes(rows, read_options=opts)

# Can skip rows with a different number of columns

rows = b"abcd\n,,,,,\nij,kl\nmn,op\n"

opts.skip_rows = 2

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["ij", "kl"])

assert table.to_pydict() == {

"ij": ["mn"],

"kl": ["op"],

}

# Can skip all rows exactly when columns are given

opts.skip_rows = 4

opts.column_names = ['ij', 'kl']

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["ij", "kl"])

assert table.to_pydict() == {

"ij": [],

"kl": [],

}

def test_skip_rows_after_names(self):

rows = b"ab,cd\nef,gh\nij,kl\nmn,op\n"

opts = ReadOptions()

opts.skip_rows_after_names = 1

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["ab", "cd"])

assert table.to_pydict() == {

"ab": ["ij", "mn"],

"cd": ["kl", "op"],

}

# Can skip exact number of rows

opts.skip_rows_after_names = 3

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["ab", "cd"])

assert table.to_pydict() == {

"ab": [],

"cd": [],

}

# Can skip beyond all rows

opts.skip_rows_after_names = 4

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["ab", "cd"])

assert table.to_pydict() == {

"ab": [],

"cd": [],

}

# Can skip rows with a different number of columns

rows = b"abcd\n,,,,,\nij,kl\nmn,op\n"

opts.skip_rows_after_names = 2

opts.column_names = ["f0", "f1"]

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["f0", "f1"])

assert table.to_pydict() == {

"f0": ["ij", "mn"],

"f1": ["kl", "op"],

}

opts = ReadOptions()

# Can skip rows with new lines in the value

rows = b'ab,cd\n"e\nf","g\n\nh"\n"ij","k\nl"\nmn,op'

opts.skip_rows_after_names = 2

parse_opts = ParseOptions()

parse_opts.newlines_in_values = True

table = self.read_bytes(rows, read_options=opts,

parse_options=parse_opts)

self.check_names(table, ["ab", "cd"])

assert table.to_pydict() == {

"ab": ["mn"],

"cd": ["op"],

}

# Can skip rows when block ends in middle of quoted value

opts.skip_rows_after_names = 2

opts.block_size = 26

table = self.read_bytes(rows, read_options=opts,

parse_options=parse_opts)

self.check_names(table, ["ab", "cd"])

assert table.to_pydict() == {

"ab": ["mn"],

"cd": ["op"],

}

opts = ReadOptions()

# Can skip rows that are beyond the first block without lexer

rows, expected = make_random_csv(num_cols=5, num_rows=1000)

opts.skip_rows_after_names = 900

opts.block_size = len(rows) / 11

table = self.read_bytes(rows, read_options=opts)

assert table.schema == expected.schema

assert table.num_rows == 100

table_dict = table.to_pydict()

for name, values in expected.to_pydict().items():

assert values[900:] == table_dict[name]

# Can skip rows that are beyond the first block with lexer

table = self.read_bytes(rows, read_options=opts,

parse_options=parse_opts)

assert table.schema == expected.schema

assert table.num_rows == 100

table_dict = table.to_pydict()

for name, values in expected.to_pydict().items():

assert values[900:] == table_dict[name]

# Skip rows and skip rows after names

rows, expected = make_random_csv(num_cols=5, num_rows=200,

write_names=False)

opts = ReadOptions()

opts.skip_rows = 37

opts.skip_rows_after_names = 41

opts.column_names = expected.schema.names

table = self.read_bytes(rows, read_options=opts,

parse_options=parse_opts)

assert table.schema == expected.schema

assert (table.num_rows ==

expected.num_rows - opts.skip_rows -

opts.skip_rows_after_names)

table_dict = table.to_pydict()

for name, values in expected.to_pydict().items():

assert (values[opts.skip_rows + opts.skip_rows_after_names:] ==

table_dict[name])

def test_row_number_offset_in_errors(self):

# Row numbers are only correctly counted in serial reads

def format_msg(msg_format, row, *args):

if self.use_threads:

row_info = ""

else:

row_info = "Row #{}: ".format(row)

return msg_format.format(row_info, *args)

csv, _ = make_random_csv(4, 100, write_names=True)

read_options = ReadOptions()

read_options.block_size = len(csv) / 3

convert_options = ConvertOptions()

convert_options.column_types = {"a": pa.int32()}

# Test without skip_rows and column names in the csv

csv_bad_columns = csv + b"1,2\r\n"

message_columns = format_msg("{}Expected 4 columns, got 2", 102)

with pytest.raises(pa.ArrowInvalid, match=message_columns):

self.read_bytes(csv_bad_columns,

read_options=read_options,

convert_options=convert_options)

csv_bad_type = csv + b"a,b,c,d\r\n"

message_value = format_msg(

"In CSV column #0: {}"

"CSV conversion error to int32: invalid value 'a'",

102, csv)

with pytest.raises(pa.ArrowInvalid, match=message_value):

self.read_bytes(csv_bad_type,

read_options=read_options,

convert_options=convert_options)

long_row = (b"this is a long row" * 15) + b",3\r\n"

csv_bad_columns_long = csv + long_row

message_long = format_msg("{}Expected 4 columns, got 2: {} ...", 102,

long_row[0:96].decode("utf-8"))

with pytest.raises(pa.ArrowInvalid, match=message_long):

self.read_bytes(csv_bad_columns_long,

read_options=read_options,

convert_options=convert_options)

# Test skipping rows after the names

read_options.skip_rows_after_names = 47

with pytest.raises(pa.ArrowInvalid, match=message_columns):

self.read_bytes(csv_bad_columns,

read_options=read_options,

convert_options=convert_options)

with pytest.raises(pa.ArrowInvalid, match=message_value):

self.read_bytes(csv_bad_type,

read_options=read_options,

convert_options=convert_options)

with pytest.raises(pa.ArrowInvalid, match=message_long):

self.read_bytes(csv_bad_columns_long,

read_options=read_options,

convert_options=convert_options)

read_options.skip_rows_after_names = 0

# Test without skip_rows and column names not in the csv

csv, _ = make_random_csv(4, 100, write_names=False)

read_options.column_names = ["a", "b", "c", "d"]

csv_bad_columns = csv + b"1,2\r\n"

message_columns = format_msg("{}Expected 4 columns, got 2", 101)

with pytest.raises(pa.ArrowInvalid, match=message_columns):

self.read_bytes(csv_bad_columns,

read_options=read_options,

convert_options=convert_options)

csv_bad_columns_long = csv + long_row

message_long = format_msg("{}Expected 4 columns, got 2: {} ...", 101,

long_row[0:96].decode("utf-8"))

with pytest.raises(pa.ArrowInvalid, match=message_long):

self.read_bytes(csv_bad_columns_long,

read_options=read_options,

convert_options=convert_options)

csv_bad_type = csv + b"a,b,c,d\r\n"

message_value = format_msg(

"In CSV column #0: {}"

"CSV conversion error to int32: invalid value 'a'",

101)

message_value = message_value.format(len(csv))

with pytest.raises(pa.ArrowInvalid, match=message_value):

self.read_bytes(csv_bad_type,

read_options=read_options,

convert_options=convert_options)

# Test with skip_rows and column names not in the csv

read_options.skip_rows = 23

with pytest.raises(pa.ArrowInvalid, match=message_columns):

self.read_bytes(csv_bad_columns,

read_options=read_options,

convert_options=convert_options)

with pytest.raises(pa.ArrowInvalid, match=message_value):

self.read_bytes(csv_bad_type,

read_options=read_options,

convert_options=convert_options)

def test_invalid_row_handler(self):

rows = b"a,b\nc\nd,e\nf,g,h\ni,j\n"

parse_opts = ParseOptions()

with pytest.raises(

ValueError,

match="Expected 2 columns, got 1: c"):

self.read_bytes(rows, parse_options=parse_opts)

# Skip requested

parse_opts.invalid_row_handler = InvalidRowHandler('skip')

table = self.read_bytes(rows, parse_options=parse_opts)

assert table.to_pydict() == {

'a': ["d", "i"],

'b': ["e", "j"],

}

def row_num(x):

return None if self.use_threads else x

expected_rows = [

InvalidRow(2, 1, row_num(2), "c"),

InvalidRow(2, 3, row_num(4), "f,g,h"),

]

assert parse_opts.invalid_row_handler.rows == expected_rows

# Error requested

parse_opts.invalid_row_handler = InvalidRowHandler('error')

with pytest.raises(

ValueError,

match="Expected 2 columns, got 1: c"):

self.read_bytes(rows, parse_options=parse_opts)

expected_rows = [InvalidRow(2, 1, row_num(2), "c")]

assert parse_opts.invalid_row_handler.rows == expected_rows

# Test ser/de

parse_opts.invalid_row_handler = InvalidRowHandler('skip')

parse_opts = pickle.loads(pickle.dumps(parse_opts))

table = self.read_bytes(rows, parse_options=parse_opts)

assert table.to_pydict() == {

'a': ["d", "i"],

'b': ["e", "j"],

def read_csv(self, csv, *args, validate_full=True, **kwargs):

"""

assert isinstance(self.use_threads, bool) # sanity check

read_options = kwargs.setdefault('read_options', ReadOptions())

read_options.use_threads = self.use_threads

table = read_csv(csv, *args, **kwargs)

table.validate(full=validate_full)

return table

def read_bytes(self, b, **kwargs):

return self.read_csv(pa.py_buffer(b), **kwargs)

def test_file_object(self):

data = b"a,b\n1,2\n"

expected_data = {'a': [1], 'b': [2]}

bio = io.BytesIO(data)

table = self.read_csv(bio)

assert table.to_pydict() == expected_data

# Text files not allowed

sio = io.StringIO(data.decode())

with pytest.raises(TypeError):

self.read_csv(sio)

def test_header(self):

rows = b"abc,def,gh\n"

table = self.read_bytes(rows)

assert isinstance(table, pa.Table)

self.check_names(table, ["abc", "def", "gh"])

assert table.num_rows == 0

def test_bom(self):

rows = b"\xef\xbb\xbfa,b\n1,2\n"

expected_data = {'a': [1], 'b': [2]}

table = self.read_bytes(rows)

assert table.to_pydict() == expected_data

def test_one_chunk(self):

# ARROW-7661: lack of newline at end of file should not produce

# an additional chunk.

rows = [b"a,b", b"1,2", b"3,4", b"56,78"]

for line_ending in [b'\n', b'\r', b'\r\n']:

for file_ending in [b'', line_ending]:

data = line_ending.join(rows) + file_ending

table = self.read_bytes(data)

assert len(table.to_batches()) == 1

assert table.to_pydict() == {

"a": [1, 3, 56],

"b": [2, 4, 78],

}

def test_header_column_names(self):

rows = b"ab,cd\nef,gh\nij,kl\nmn,op\n"

opts = ReadOptions()

opts.column_names = ["x", "y"]

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["x", "y"])

assert table.to_pydict() == {

"x": ["ab", "ef", "ij", "mn"],

"y": ["cd", "gh", "kl", "op"],

}

opts.skip_rows = 3

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["x", "y"])

assert table.to_pydict() == {

"x": ["mn"],

"y": ["op"],

}

opts.skip_rows = 4

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["x", "y"])

assert table.to_pydict() == {

"x": [],

"y": [],

}

opts.skip_rows = 5

with pytest.raises(pa.ArrowInvalid):

# Not enough rows

table = self.read_bytes(rows, read_options=opts)

# Unexpected number of columns

opts.skip_rows = 0

opts.column_names = ["x", "y", "z"]

with pytest.raises(pa.ArrowInvalid,

match="Expected 3 columns, got 2"):

table = self.read_bytes(rows, read_options=opts)

# Can skip rows with a different number of columns

rows = b"abcd\n,,,,,\nij,kl\nmn,op\n"

opts.skip_rows = 2

opts.column_names = ["x", "y"]

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["x", "y"])

assert table.to_pydict() == {

"x": ["ij", "mn"],

"y": ["kl", "op"],

}

def test_header_autogenerate_column_names(self):

rows = b"ab,cd\nef,gh\nij,kl\nmn,op\n"

opts = ReadOptions()

opts.autogenerate_column_names = True

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["f0", "f1"])

assert table.to_pydict() == {

"f0": ["ab", "ef", "ij", "mn"],

"f1": ["cd", "gh", "kl", "op"],

}

opts.skip_rows = 3

table = self.read_bytes(rows, read_options=opts)

self.check_names(table, ["f0", "f1"])

assert table.to_pydict() == {

"f0": ["mn"],

"f1": ["op"],

}

# Not enough rows, impossible to infer number of columns

opts.skip_rows = 4

with pytest.raises(pa.ArrowInvalid):

table = self.read_bytes(rows, read_options=opts)

def test_include_columns(self):

rows = b"ab,cd\nef,gh\nij,kl\nmn,op\n"

convert_options = ConvertOptions()

convert_options.include_columns = ['ab']

table = self.read_bytes(rows, convert_options=convert_options)

self.check_names(table, ["ab"])

assert table.to_pydict() == {

"ab": ["ef", "ij", "mn"],

}

# Order of include_columns is respected, regardless of CSV order

convert_options.include_columns = ['cd', 'ab']

table = self.read_bytes(rows, convert_options=convert_options)

schema = pa.schema([('cd', pa.string()),

('ab', pa.string())])

assert table.schema == schema

assert table.to_pydict() == {

"cd": ["gh", "kl", "op"],

"ab": ["ef", "ij", "mn"],

}

# Include a column not in the CSV file => raises by default

convert_options.include_columns = ['xx', 'ab', 'yy']

with pytest.raises(KeyError,

match="Column 'xx' in include_columns "

"does not exist in CSV file"):

self.read_bytes(rows, convert_options=convert_options)

def test_include_missing_columns(self):

rows = b"ab,cd\nef,gh\nij,kl\nmn,op\n"

read_options = ReadOptions()

convert_options = ConvertOptions()

convert_options.include_columns = ['xx', 'ab', 'yy']

convert_options.include_missing_columns = True

table = self.read_bytes(rows, read_options=read_options,

convert_options=convert_options)

schema = pa.schema([('xx', pa.null()),

('ab', pa.string()),

('yy', pa.null())])

assert table.schema == schema

assert table.to_pydict() == {

"xx": [None, None, None],

"ab": ["ef", "ij", "mn"],

"yy": [None, None, None],

}

# Combining with `column_names`

read_options.column_names = ["xx", "yy"]

convert_options.include_columns = ["yy", "cd"]

table = self.read_bytes(rows, read_options=read_options,

convert_options=convert_options)

schema = pa.schema([('yy', pa.string()),

('cd', pa.null())])

assert table.schema == schema

assert table.to_pydict() == {

"yy": ["cd", "gh", "kl", "op"],

"cd": [None, None, None, None],

}

# And with `column_types` as well

convert_options.column_types = {"yy": pa.binary(),

"cd": pa.int32()}

table = self.read_bytes(rows, read_options=read_options,

convert_options=convert_options)

schema = pa.schema([('yy', pa.binary()),

('cd', pa.int32())])

assert table.schema == schema

assert table.to_pydict() == {

"yy": [b"cd", b"gh", b"kl", b"op"],

"cd": [None, None, None, None],

}

def test_simple_ints(self):

# Infer integer columns

rows = b"a,b,c\n1,2,3\n4,5,6\n"

table = self.read_bytes(rows)

schema = pa.schema([('a', pa.int64()),

('b', pa.int64()),

('c', pa.int64())])

assert table.schema == schema

assert table.to_pydict() == {

'a': [1, 4],

'b': [2, 5],

'c': [3, 6],

}

def test_simple_varied(self):

# Infer various kinds of data

rows = b"a,b,c,d\n1,2,3,0\n4.0,-5,foo,True\n"

table = self.read_bytes(rows)

schema = pa.schema([('a', pa.float64()),

('b', pa.int64()),

('c', pa.string()),

('d', pa.bool_())])

assert table.schema == schema

assert table.to_pydict() == {

'a': [1.0, 4.0],

'b': [2, -5],

'c': ["3", "foo"],

'd': [False, True],

}

def test_simple_nulls(self):

# Infer various kinds of data, with nulls

rows = (b"a,b,c,d,e,f\n"

b"1,2,,,3,N/A\n"

b"nan,-5,foo,,nan,TRUE\n"

b"4.5,#N/A,nan,,\xff,false\n")

table = self.read_bytes(rows)

schema = pa.schema([('a', pa.float64()),

('b', pa.int64()),

('c', pa.string()),

('d', pa.null()),

('e', pa.binary()),

('f', pa.bool_())])

assert table.schema == schema

assert table.to_pydict() == {

'a': [1.0, None, 4.5],

'b': [2, -5, None],

'c': ["", "foo", "nan"],

'd': [None, None, None],

'e': [b"3", b"nan", b"\xff"],

'f': [None, True, False],

}

def test_decimal_point(self):

# Infer floats with a custom decimal point

parse_options = ParseOptions(delimiter=';')

rows = b"a;b\n1.25;2,5\nNA;-3\n-4;NA"

table = self.read_bytes(rows, parse_options=parse_options)

schema = pa.schema([('a', pa.float64()),

('b', pa.string())])

assert table.schema == schema

assert table.to_pydict() == {

'a': [1.25, None, -4.0],

'b': ["2,5", "-3", "NA"],

}

convert_options = ConvertOptions(decimal_point=',')

table = self.read_bytes(rows, parse_options=parse_options,

convert_options=convert_options)

schema = pa.schema([('a', pa.string()),

('b', pa.float64())])

assert table.schema == schema

assert table.to_pydict() == {

'a': ["1.25", "NA", "-4"],

'b': [2.5, -3.0, None],

}

def test_simple_timestamps(self):

# Infer a timestamp column

rows = (b"a,b,c\n"

b"1970,1970-01-01 00:00:00,1970-01-01 00:00:00.123\n"

b"1989,1989-07-14 01:00:00,1989-07-14 01:00:00.123456\n")

table = self.read_bytes(rows)

schema = pa.schema([('a', pa.int64()),

('b', pa.timestamp('s')),

('c', pa.timestamp('ns'))])

assert table.schema == schema

assert table.to_pydict() == {

'a': [1970, 1989],

'b': [datetime(1970, 1, 1), datetime(1989, 7, 14, 1)],

'c': [datetime(1970, 1, 1, 0, 0, 0, 123000),

datetime(1989, 7, 14, 1, 0, 0, 123456)],

}

def test_timestamp_parsers(self):

# Infer timestamps with custom parsers

rows = b"a,b\n1970/01/01,1980-01-01 00\n1970/01/02,1980-01-02 00\n"

opts = ConvertOptions()

table = self.read_bytes(rows, convert_options=opts)

schema = pa.schema([('a', pa.string()),

('b', pa.timestamp('s'))])

assert table.schema == schema

assert table.to_pydict() == {

'a': ['1970/01/01', '1970/01/02'],

'b': [datetime(1980, 1, 1), datetime(1980, 1, 2)],

}

opts.timestamp_parsers = ['%Y/%m/%d']

table = self.read_bytes(rows, convert_options=opts)

schema = pa.schema([('a', pa.timestamp('s')),

('b', pa.string())])

assert table.schema == schema

assert table.to_pydict() == {

'a': [datetime(1970, 1, 1), datetime(1970, 1, 2)],

'b': ['1980-01-01 00', '1980-01-02 00'],

}

opts.timestamp_parsers = ['%Y/%m/%d', ISO8601]

table = self.read_bytes(rows, convert_options=opts)

schema = pa.schema([('a', pa.timestamp('s')),

('b', pa.timestamp('s'))])

assert table.schema == schema