Skip to content

fix: Widen Athena integer type mapping for unsigned ints #6739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
nikolauspschuetz wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

fix: Widen Athena integer type mapping for unsigned ints #6739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions sdk/python/feast/type_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2291,10 +2291,14 @@ def pa_to_athena_value_type(pa_type: "pyarrow.DataType") -> str:
"int16": "smallint",
"int32": "int",
"int64": "bigint",
"uint8": "tinyint",
"uint16": "tinyint",
"uint32": "tinyint",
"uint64": "tinyint",
# Athena integer types are signed, so unsigned Arrow types must be
# widened to the next-larger signed type to avoid overflow in the
# generated DDL (e.g. uint32 exceeds signed int's max). This mirrors
# the widening already done in arrow_to_pg_type for Postgres.
"uint8": "smallint",
"uint16": "int",
"uint32": "bigint",
"uint64": "bigint",
"float": "float",
"double": "double",
"binary": "binary",
Expand Down
20 changes: 20 additions & 0 deletions sdk/python/tests/unit/test_type_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
arrow_to_pg_type,
feast_value_type_to_pa,
feast_value_type_to_python_type,
pa_to_athena_value_type,
pa_to_feast_value_type,
pa_to_redshift_value_type,
pg_type_to_feast_value_type,
Expand Down Expand Up @@ -531,6 +532,25 @@ def test_arrow_to_pg_type_map(self):
assert arrow_to_pg_type("map<string, string>") == "jsonb"
assert arrow_to_pg_type("map<string, int64>") == "jsonb"

def test_pa_to_athena_value_type_unsigned_ints_widen(self):
"""Unsigned Arrow ints must widen to a signed Athena type that can
hold their full range. Athena has no unsigned integer types, so
mapping every uintN to tinyint (signed -128..127) overflows for any
value above 127 in the generated CREATE TABLE DDL. Each uintN must map
to the next-larger signed type, matching arrow_to_pg_type's widening.
"""
assert pa_to_athena_value_type(pyarrow.uint8()) == "smallint"
assert pa_to_athena_value_type(pyarrow.uint16()) == "int"
assert pa_to_athena_value_type(pyarrow.uint32()) == "bigint"
assert pa_to_athena_value_type(pyarrow.uint64()) == "bigint"

def test_pa_to_athena_value_type_signed_ints_unchanged(self):
"""Signed Arrow ints keep their same-width Athena type (no regression)."""
assert pa_to_athena_value_type(pyarrow.int8()) == "tinyint"
assert pa_to_athena_value_type(pyarrow.int16()) == "smallint"
assert pa_to_athena_value_type(pyarrow.int32()) == "int"
assert pa_to_athena_value_type(pyarrow.int64()) == "bigint"

def test_pg_type_to_feast_value_type_json(self):
"""Test that Postgres json/jsonb types convert to ValueType.MAP."""
assert pg_type_to_feast_value_type("json") == ValueType.MAP
Expand Down
Loading