Skip to content

PERF: Slow performance of to_dict (#46470) #46487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rhshadrach merged 17 commits into from
Nov 22, 2022
Merged

PERF: Slow performance of to_dict (#46470) #46487

Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
8d93fec
PERF: Slow performance of to_dict (#46470)
Apr 22, 2022
d9f9786
Update
Apr 22, 2022
96ac6fa
Update
Apr 22, 2022
3c596f7
Clean up
Apr 22, 2022
57c95eb
Address PR comments
Apr 24, 2022
e07f02c
Address PR comments
Apr 29, 2022
d2da86b
Use as set
Apr 29, 2022
0c0481d
Add helper function
May 10, 2022
1d68f83
Add comment and types
May 11, 2022
b413f99
Merge branch 'main' of https://github.com/pandas-dev/pandas into perf…
Jun 27, 2022
f66340d
Return quickly if series
Jun 27, 2022
03978aa
Add return value
Jun 28, 2022
e1aee52
Merge branch 'main' of https://github.com/pandas-dev/pandas into perf…
Aug 17, 2022
aa9863d
Merge branch 'main' of https://github.com/pandas-dev/pandas into perf…
Nov 1, 2022
ecfffa6
Move doc to 2.0.0
Nov 22, 2022
6487cc9
Merge branch 'main' of https://github.com/pandas-dev/pandas into perf…
Nov 22, 2022
36e22c0
Revert
Nov 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add helper function
  • Loading branch information
Roger Thomas committed May 10, 2022
commit 0c0481d25dbd00882154954e95d145f7193df08a
52 changes: 26 additions & 26 deletions pandas/core/frame.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1771,6 +1771,24 @@ def to_numpy(

return result

def _create_data_for_split_and_tight_to_dict(
self, are_all_object_dtype_cols, object_dtype_indices
):
if are_all_object_dtype_cols:
data = [
list(map(maybe_box_native, t))
for t in self.itertuples(index=False, name=None)
]
else:
data = [list(t) for t in self.itertuples(index=False, name=None)]
if object_dtype_indices:
# If we have object_dtype_cols, apply maybe_box_naive after list
# comprehension for perf
for row in data:
for i in object_dtype_indices:
row[i] = maybe_box_native(row[i])
return data

def to_dict(self, orient: str = "dict", into=dict):
"""
Convert the DataFrame to a dictionary.
Expand Down Expand Up @@ -1932,19 +1950,10 @@ def to_dict(self, orient: str = "dict", into=dict):
)

elif orient == "split":
if are_all_object_dtype_cols:
data = [
list(map(maybe_box_native, t))
for t in self.itertuples(index=False, name=None)
]
else:
data = [list(t) for t in self.itertuples(index=False, name=None)]
if object_dtype_indices:
# If we have object_dtype_cols, apply maybe_box_naive after list
# comprehension for perf
for row in data:
for i in object_dtype_indices:
row[i] = maybe_box_native(row[i])
data = self._create_data_for_split_and_tight_to_dict(
are_all_object_dtype_cols, object_dtype_indices
)

return into_c(
(
("index", self.index.tolist()),
Expand All @@ -1954,19 +1963,10 @@ def to_dict(self, orient: str = "dict", into=dict):
)

elif orient == "tight":
if are_all_object_dtype_cols:
data = [
list(map(maybe_box_native, t))
for t in self.itertuples(index=False, name=None)
]
else:
data = [list(t) for t in self.itertuples(index=False, name=None)]
if object_dtype_indices:
# If we have object_dtype_cols, apply maybe_box_naive after list
# comprehension for perf
for row in data:
for i in object_dtype_indices:
row[i] = maybe_box_native(row[i])
data = self._create_data_for_split_and_tight_to_dict(
are_all_object_dtype_cols, object_dtype_indices
)

return into_c(
(
("index", self.index.tolist()),
Expand Down