Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 188 additions & 58 deletions Lib/heapq.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,68 +329,198 @@ def merge(*iterables, key=None, reverse=False):

'''

h = []
h_append = h.append
n = len(iterables)
if not n:
return
n_1 = n - 1

if reverse:
_heapify = _heapify_max
_heappop = _heappop_max
_heapreplace = _heapreplace_max
direction = -1
else:
_heapify = heapify
_heappop = heappop
_heapreplace = heapreplace
direction = 1
# shift the iterators so that the first iterable passed will
# correspond to the leftmost node
shift = n - (1 << n.bit_length())
iters = [iter(iterables[i+shift]) for i in range(n)]

if key is None:
for order, it in enumerate(map(iter, iterables)):
try:
next = it.__next__
h_append([next(), order * direction, next])
except StopIteration:
pass
_heapify(h)
while len(h) > 1:
try:
while True:
value, order, next = s = h[0]
yield value
s[0] = next() # raises StopIteration when exhausted
_heapreplace(h, s) # restore heap condition
except StopIteration:
_heappop(h) # remove empty iterator
if h:
# fast case when only a single iterator remains
value, order, next = h[0]
yield value
yield from next.__self__
return
# _tree will be a binary heap structure, but we won't do the usual
# heap operations. Instead, only do _treesift_func: items will only
# ever move toward the root, and the leaves are supplied by the
# iterators. This a non-recursive implementation of recursively
# applying a 2-iterator merge.

for order, it in enumerate(map(iter, iterables)):
try:
next = it.__next__
value = next()
h_append([key(value), order * direction, value, next])
except StopIteration:
pass
_heapify(h)
while len(h) > 1:
try:
while True:
key_value, order, value, next = s = h[0]
yield value
value = next()
s[0] = key(value)
s[2] = value
_heapreplace(h, s)
except StopIteration:
_heappop(h)
if h:
key_value, order, value, next = h[0]
yield value
yield from next.__self__
tree = [None] * (n+n_1)
sentinel = object()
_next = next
_StopIteration = StopIteration

if key is None:
if reverse:
# Case 1: No key, reversed
for pos in reversed(range(len(tree))):
# heapify, except draw new items from the iterators
# rather than from swaps.
while pos < n_1:
childpos = 2 * pos + 1
child = tree[childpos]
otherchild = tree[childpos + 1]
if otherchild is not sentinel \
and (child is sentinel or child < otherchild):
childpos += 1
child = otherchild
tree[pos] = child
pos = childpos
tree[pos] = _next(iters[pos - n_1], sentinel)
while True:
result = tree[0]
if result is sentinel:
return
yield result
pos = 0
while pos < n_1:
childpos = 2 * pos + 1
child = tree[childpos]
otherchild = tree[childpos + 1]
if otherchild is not sentinel \
and (child is sentinel or child < otherchild):
childpos += 1
child = otherchild
tree[pos] = child
pos = childpos
# this is where the shift pays off
tree[pos] = _next(iters[pos - n_1], sentinel)
else:
# Case 2: No key forward
for pos in reversed(range(len(tree))):
while pos < n_1:
childpos = 2 * pos + 1
child = tree[childpos]
otherchild = tree[childpos + 1]
if otherchild is not sentinel \
and (child is sentinel or otherchild < child):
childpos += 1
child = otherchild
tree[pos] = child
pos = childpos
tree[pos] = _next(iters[pos - n_1], sentinel)
while True:
result = tree[0]
if result is sentinel:
return
yield result
pos = 0
while pos < n_1:
childpos = 2 * pos + 1
child = tree[childpos]
otherchild = tree[childpos + 1]
if otherchild is not sentinel \
and (child is sentinel or otherchild < child):
childpos += 1
child = otherchild
tree[pos] = child
pos = childpos
tree[pos] = _next(iters[pos - n_1], sentinel)
else:
# Keys: store flatly next to their items
tree *= 2
if reverse:
# Case 3: Using a key, reverse.
for pos in reversed(range(0, len(tree), 2)):
while (pos >> 1) < n_1:
childpos = 2 * pos + 2
childkey = tree[childpos+1]
otherchildkey = tree[childpos+3]
if otherchildkey is not sentinel \
and (childkey is sentinel
or childkey < otherchildkey):
childpos += 2
childkey = otherchildkey
tree[pos+1] = childkey
tree[pos] = tree[childpos]
pos = childpos
try:
child = _next(iters[(pos >> 1) - n_1])
except _StopIteration:
child = sentinel
childkey = sentinel
else:
childkey = key(child)
tree[pos] = child
tree[pos+1] = childkey
while True:
result = tree[0]
if result is sentinel:
return
yield result
pos = 0
while (pos >> 1) < n_1:
childpos = 2 * pos + 2
childkey = tree[childpos+1]
otherchildkey = tree[childpos+3]
if otherchildkey is not sentinel \
and (childkey is sentinel
or childkey < otherchildkey):
childpos += 2
childkey = otherchildkey
tree[pos+1] = childkey
tree[pos] = tree[childpos]
pos = childpos
try:
child = _next(iters[(pos >> 1) - n_1])
except _StopIteration:
child = sentinel
childkey = sentinel
else:
childkey = key(child)
tree[pos] = child
tree[pos+1] = childkey
else:
# Case 4: Using a key, forward.
for pos in reversed(range(0, len(tree), 2)):
while (pos >> 1) < n_1:
childpos = 2 * pos + 2
childkey = tree[childpos+1]
otherchildkey = tree[childpos+3]
if otherchildkey is not sentinel \
and (childkey is sentinel
or otherchildkey < childkey):
childpos += 2
childkey = otherchildkey
tree[pos+1] = childkey
tree[pos] = tree[childpos]
pos = childpos
try:
child = _next(iters[(pos >> 1) - n_1])
except _StopIteration:
child = sentinel
childkey = sentinel
else:
childkey = key(child)
tree[pos] = child
tree[pos+1] = childkey
while True:
result = tree[0]
if result is sentinel:
return
yield result
pos = 0
while (pos >> 1) < n_1:
childpos = 2 * pos + 2
childkey = tree[childpos+1]
otherchildkey = tree[childpos+3]
if otherchildkey is not sentinel \
and (childkey is sentinel
or otherchildkey < childkey):
childpos += 2
childkey = otherchildkey
tree[pos+1] = childkey
tree[pos] = tree[childpos]
pos = childpos
try:
child = _next(iters[(pos >> 1) - n_1])
except _StopIteration:
child = sentinel
childkey = sentinel
else:
childkey = key(child)
tree[pos] = child
tree[pos+1] = childkey

# Algorithm notes for nlargest() and nsmallest()
# ==============================================
Expand Down
35 changes: 35 additions & 0 deletions Lib/test/test_heapq.py
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,41 @@ def __lt__(self, o):
self.assertRaises((IndexError, RuntimeError), self.module.heappush, list1, g(1))
self.assertRaises((IndexError, RuntimeError), self.module.heappush, list2, h(1))

def test_merge_err(self):
nexterr_immediate = E
def nexterr_delayed():
yield from range(10)
raise ZeroDivisionError
merge = self.module.merge
for key in [None, lambda x:x]:
for reverse in [False, True]:
# test comparison failure during heapification
args = [[CmpErr()]] + [range(100)]*10
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)

for n in range(2,10):
# test comparison failure during intermediate steps
args = [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, CmpErr()]]*n
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
for n in range(1, 10):
# test __next__ error during heapification
args = [nexterr_immediate(range(10)) for _ in range(n)]
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
for n in range(1, 10):
# test __next__ error during intermediate steps
args = [nexterr_delayed() for _ in range(n)]
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
for reverse in [False, True]:
# test error during key computation
mo = merge(range(10), range(10), key=lambda x: x // 0, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
mo = merge(range(10), key=object(), reverse=reverse)
self.assertRaises(TypeError, list, merge)

class TestErrorHandlingPython(TestErrorHandling, TestCase):
module = py_heapq

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Changed the :func:`heap.merge` algorithm to reduce the number of
comparisons, added a C accelerator.
Loading