Skip to content

ENH: NumericIndex for any numpy int/uint/float dtype #41153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jreback merged 55 commits into from
Aug 5, 2021
Merged

ENH: NumericIndex for any numpy int/uint/float dtype #41153

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
9739f85
ENH: Add NumIndex for indexic of any numeric type
topper-123 Apr 25, 2021
88a7858
fix various issues reported by the CI
topper-123 Apr 25, 2021
7ccb1b7
fix test failure
topper-123 Apr 26, 2021
b45500d
Make (Int|UInt|Float)64Index inherit from NumIndex
topper-123 Apr 27, 2021
5ef35f5
fix errors
topper-123 Apr 27, 2021
d8f6c22
Add more numeric tests for NumIndex
topper-123 May 2, 2021
1c65a0b
fixups
topper-123 May 2, 2021
f3e13aa
fix exact='equiv'
topper-123 May 6, 2021
4e17485
add more comprehensive tests
topper-123 May 10, 2021
c1e801d
fixes
topper-123 May 10, 2021
aa0cea7
addresses comments (move _format_native_types, assert_index_equal etc.)
topper-123 May 10, 2021
1f5f922
remove from public namespace
topper-123 May 10, 2021
132ce44
rename to NumericIndex
topper-123 May 10, 2021
058cd2e
fixes
topper-123 May 10, 2021
1c7f23f
fix test
topper-123 May 10, 2021
07a097c
ENH: Add NumIndex for indexic of any numeric type
topper-123 Apr 25, 2021
ff6cfb4
fix test failure
topper-123 Apr 26, 2021
fe7b97c
Add more numeric tests for NumIndex
topper-123 May 2, 2021
86f3960
fixups
topper-123 May 2, 2021
2424c0d
fix exact='equiv'
topper-123 May 6, 2021
a515bba
add more comprehensive tests
topper-123 May 10, 2021
341fc2f
remove from public namespace
topper-123 May 10, 2021
c2d8884
fix test
topper-123 May 10, 2021
5a56b1a
add back numeric tests
topper-123 May 11, 2021
a497d57
fix comments
topper-123 May 12, 2021
6557689
fix comments part II
topper-123 May 12, 2021
5bc4c2c
_is_num_index -> _is_numeric_index + Index.union
topper-123 May 14, 2021
84bf540
makeNumIndex -> makeNumericIndex and refactor makeIntIndex etc.
topper-123 May 14, 2021
69953b4
fix errors
topper-123 May 20, 2021
b4be77d
rebase after #41472
topper-123 May 20, 2021
bb42e2d
small clean-up
topper-123 May 20, 2021
bafa9b3
small clean-up II
topper-123 May 20, 2021
35b0e71
small clean-up III
topper-123 May 20, 2021
ed4730b
small clean-up IV
topper-123 May 20, 2021
6a32788
small clean-up V
topper-123 May 20, 2021
47e208c
fix bug
topper-123 May 20, 2021
d6a03a0
fix failures
topper-123 May 21, 2021
ec003ed
cleanups
topper-123 May 21, 2021
7ddee71
chabge _is_numeric_index to be an attribute
topper-123 May 21, 2021
2bb282f
minor clean-ups
topper-123 May 21, 2021
c1633fb
fix not-allowed parameter
topper-123 May 21, 2021
9c7d57b
fix _should_fallback_to_positional
topper-123 May 22, 2021
f6dccc1
clean-ups after rebasing
topper-123 Jun 4, 2021
3630fc7
more clean-ups
topper-123 Jun 4, 2021
bfe6895
add cleanups
topper-123 Jun 5, 2021
8532ddb
fix TestApi failure
topper-123 Jun 5, 2021
186de8e
more precise tests
topper-123 Jun 5, 2021
ead8f57
update tests
topper-123 Jun 21, 2021
2a850ea
update asserters doc string
topper-123 Jun 21, 2021
d04da70
update tests/common.py
topper-123 Jun 21, 2021
4b8385c
cleanups
topper-123 Jun 23, 2021
1f52f8b
simplify _ensure_dtype
topper-123 Jul 24, 2021
951c5f7
make attribute name clearer
topper-123 Jul 25, 2021
7c7c0dd
address comments
topper-123 Jul 28, 2021
bb72c68
add TODO
topper-123 Jul 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rename to NumericIndex
  • Loading branch information
@topper-123
topper-123 committed Jul 31, 2021
commit 132ce4401ed164e81e13a37c219a7128fb2eb18d
2 changes: 1 addition & 1 deletion pandas/_testing/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@
use_numexpr,
with_csv_dialect,
)
from pandas.core.api import NumIndex
from pandas.core.api import NumericIndex
from pandas.core.arrays import (
DatetimeArray,
PandasArray,
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
Int64Index,
IntervalIndex,
MultiIndex,
NumIndex,
NumericIndex,
PeriodIndex,
RangeIndex,
TimedeltaIndex,
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import warnings

from pandas.core.indexes.api import ( # noqa:F401
BaseNumericIndex,
CategoricalIndex,
DatetimeIndex,
Float64Index,
Expand All @@ -9,7 +10,6 @@
IntervalIndex,
MultiIndex,
NaT,
NumericIndex,
PeriodIndex,
RangeIndex,
TimedeltaIndex,
Expand Down
4 changes: 2 additions & 2 deletions pandas/core/indexes/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
from pandas.core.indexes.interval import IntervalIndex
from pandas.core.indexes.multi import MultiIndex
from pandas.core.indexes.numeric import (
BaseNumericIndex,
Float64Index,
Int64Index,
NumericIndex,
NumIndex,
UInt64Index,
)
from pandas.core.indexes.period import PeriodIndex
Expand All @@ -45,8 +45,8 @@
__all__ = [
"Index",
"MultiIndex",
"NumIndex",
"NumericIndex",
"BaseNumericIndex",
"Float64Index",
"Int64Index",
"CategoricalIndex",
Expand Down
14 changes: 7 additions & 7 deletions pandas/core/indexes/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ def __new__(
)

from pandas.core.arrays import PandasArray
from pandas.core.indexes.numeric import NumIndex
from pandas.core.indexes.numeric import NumericIndex
from pandas.core.indexes.range import RangeIndex

name = maybe_extract_name(name, data, cls)
Expand Down Expand Up @@ -438,8 +438,8 @@ def __new__(
return Index._simple_new(data, name=name)

# index-like
elif isinstance(data, NumIndex) and data._is_num_index() and dtype is None:
return NumIndex(data, name=name, copy=copy)
elif isinstance(data, NumericIndex) and data._is_num_index() and dtype is None:
return NumericIndex(data, name=name, copy=copy)
elif isinstance(data, (np.ndarray, Index, ABCSeries)):

if isinstance(data, ABCMultiIndex):
Expand Down Expand Up @@ -2438,11 +2438,11 @@ def _is_num_index(self) -> bool:
from pandas.core.indexes.numeric import (
Float64Index,
Int64Index,
NumIndex,
NumericIndex,
UInt64Index,
)

if not isinstance(self, NumIndex):
if not isinstance(self, NumericIndex):
return False
elif isinstance(self, (Int64Index, UInt64Index, Float64Index)):
return False
Expand Down Expand Up @@ -5723,7 +5723,7 @@ def map(self, mapper, na_action=None):
a MultiIndex will be returned.
"""
from pandas.core.indexes.multi import MultiIndex
from pandas.core.indexes.numeric import NumIndex
from pandas.core.indexes.numeric import NumericIndex

new_values = self._map_values(mapper, na_action=na_action)

Expand All @@ -5745,7 +5745,7 @@ def map(self, mapper, na_action=None):
attributes["dtype"] = self.dtype

if self._is_num_index() and is_numeric_dtype(new_values.dtype):
return NumIndex(new_values, **attributes)
return NumericIndex(new_values, **attributes)

return Index(new_values, **attributes)

Expand Down
4 changes: 2 additions & 2 deletions pandas/core/indexes/category.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ def _is_dtype_compat(self, other) -> Categorical:

@doc(Index.astype)
def astype(self, dtype, copy: bool = True) -> Index:
from pandas.core.api import NumIndex
from pandas.core.api import NumericIndex
Copy link
Member

@jbrockmendel jbrockmendel Jul 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can annotate dtype as Dtype

Copy link
Contributor Author

@topper-123 topper-123 Jul 28, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok.

Copy link
Member

@jbrockmendel jbrockmendel Jul 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we change/deprecate Index.astype(np.int32), also Index(foo, dtype=np.int32)?

Copy link
Contributor Author

@topper-123 topper-123 Jul 29, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, Index.astype(np.int32) returning an Int64Index is probably used internally, so a deprecation would require if/else clauses at various locations in the code base. Probably possible, but could we get this PR in first? This PR isn't public-facing, so I don't see a problem just merging this first/separately.

Copy link
Member

@jbrockmendel jbrockmendel Jul 30, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

doing that as a follow-up sounds good


dtype = pandas_dtype(dtype)

Expand All @@ -292,7 +292,7 @@ def astype(self, dtype, copy: bool = True) -> Index:
# but if e.g. the categories are a NumIndex with dtype float32, we want to
# return an index with the same dtype as self.categories.
if cat._is_num_index():
assert isinstance(cat, NumIndex) # mypy complaint fix
assert isinstance(cat, NumericIndex) # mypy complaint fix
try:
cat._validate_dtype(dtype)
except ValueError:
Expand Down
4 changes: 2 additions & 2 deletions pandas/core/indexes/range.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@
import pandas.core.indexes.base as ibase
from pandas.core.indexes.base import maybe_extract_name
from pandas.core.indexes.numeric import (
BaseNumericIndex,
Float64Index,
Int64Index,
NumericIndex,
)
from pandas.core.ops.common import unpack_zerodim_and_defer

Expand All @@ -57,7 +57,7 @@
_empty_range = range(0)


class RangeIndex(NumericIndex):
class RangeIndex(BaseNumericIndex):
"""
Immutable Index implementing a monotonic integer range.

Expand Down
4 changes: 2 additions & 2 deletions pandas/tests/indexes/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)
import pandas._testing as tm
from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin
from pandas.core.indexes.numeric import NumIndex
from pandas.core.indexes.numeric import NumericIndex


class Base:
Expand Down Expand Up @@ -666,7 +666,7 @@ def test_map_dictlike(self, mapper, simple_index):

# empty mappable
if idx._is_num_index():
new_index_cls = NumIndex
new_index_cls = NumericIndex
else:
new_index_cls = Float64Index

Expand Down
2 changes: 1 addition & 1 deletion pandas/tests/indexes/numeric/test_numeric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
UInt64Index,
)
import pandas._testing as tm
from pandas.core.api import NumIndex
from pandas.core.api import NumericIndex
from pandas.tests.indexes.common import NumericBase


Expand Down
6 changes: 3 additions & 3 deletions pandas/tests/indexes/test_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
TimedeltaIndex,
)
import pandas._testing as tm
from pandas.core.api import NumIndex
from pandas.core.api import NumericIndex


class TestCommon:
Expand Down Expand Up @@ -262,7 +262,7 @@ def test_drop_duplicates(self, index_flat, keep):
# make unique index
holder = type(index)
unique_values = list(set(index))
dtype = index.dtype if isinstance(index, NumIndex) else None
dtype = index.dtype if isinstance(index, NumericIndex) else None
unique_idx = holder(unique_values, dtype=dtype)

# make duplicated index
Expand Down Expand Up @@ -291,7 +291,7 @@ def test_drop_duplicates_no_duplicates(self, index_flat):
else:
holder = type(index)
unique_values = list(set(index))
dtype = index.dtype if isinstance(index, NumIndex) else None
dtype = index.dtype if isinstance(index, NumericIndex) else None
unique_idx = holder(unique_values, dtype=dtype)

# check on unique index
Expand Down
6 changes: 3 additions & 3 deletions pandas/tests/indexes/test_numpy_compat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
)
import pandas._testing as tm
from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin
from pandas.core.indexes.numeric import NumericIndex
from pandas.core.indexes.numeric import BaseNumericIndex


@pytest.mark.parametrize(
Expand Down Expand Up @@ -49,7 +49,7 @@ def test_numpy_ufuncs_basic(index, func):
with tm.external_error_raised((TypeError, AttributeError)):
with np.errstate(all="ignore"):
func(index)
elif isinstance(index, NumericIndex):
elif isinstance(index, BaseNumericIndex):
# coerces to float (e.g. np.sin)
with np.errstate(all="ignore"):
result = func(index)
Expand Down Expand Up @@ -94,7 +94,7 @@ def test_numpy_ufuncs_other(index, func, request):
with tm.external_error_raised(TypeError):
func(index)

elif isinstance(index, NumericIndex):
elif isinstance(index, BaseNumericIndex):
# Results in bool array
result = func(index)
assert isinstance(result, np.ndarray)
Expand Down