Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 140 additions & 58 deletions Lib/heapq.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,105 @@ def _siftup_max(heap, pos):
heap[pos] = newitem
_siftdown_max(heap, startpos, pos)


class _MergeNode:
"""
Binary tree invariants:
- A node N is a leaf iff N.leaf is N
- For each leaf node:
- leaf.right is an iterator
- leaf.left is the item most recently produced by leaf.right
- leaf.key is key(leaf.left)
- For each non-leaf node:
- node.left and node.right are Nodes
- node.left.parent is node is node.right.parent
- node.leaf is one of node's descendant leaves
- node.key is node.leaf.key
- if "winner" is the higher priority of node.left and
node.right based on their keys, then node.leaf is
winner.leaf and node.key is winner.key.
"""

__slots__ = "key", "leaf", "parent", "left", "right"

@classmethod
def construct_leaf(cls, iterator, keyfunc):
it = iter(iterator)
try:
item = next(it)
except StopIteration:
return None
key = item if keyfunc is None else keyfunc(item)
node = cls()
node.key = key
node.left = item
node.right = it
node.parent = None
node.leaf = node
return node

@classmethod
def construct_parent(cls, left, right, reverse):
if reverse:
winner = right if left.key < right.key else left
else:
winner = right if right.key < left.key else left
node = cls()
node.key = winner.key
node.left = left
node.right = right
node.parent = None
node.leaf = winner.leaf
left.parent = right.parent = node
return node

@classmethod
def build_tree(cls, iterables, key, reverse):
nodes = []
for it in iterables:
leaf = cls.construct_leaf(it, key)
if leaf is not None:
nodes.append(leaf)
if not nodes:
return None
n = len(nodes)
# unite pairs of adjacent nodes with a common parent until all
# nodes are united into one big tree.
while n > 1:
# Prefer keeping the leftmost nodes shallower in the tree
# since they're more likely to win for stability reasons.
new_nodes, rest = nodes[:n & 1], nodes[n & 1:]
for left, right in zip(rest[::2], rest[1::2]):
parent = cls.construct_parent(left, right, reverse)
new_nodes.append(parent)
nodes = new_nodes
n = len(nodes)
(root,) = nodes
return root

def promote_sibling(self):
"""
Remove self and its sibling from the tree, while linking their
parent to the sibling's children.
"""
assert self.leaf is self
parent = self.parent
left = parent.left
right = parent.right
sibling = left if self is right else right
parent.left = sibling.left
parent.right = sibling.right
parent.key = sibling.key
if sibling.leaf is sibling:
# sibling was a leaf, so now parent becomes a leaf
parent.leaf = parent
else:
parent.leaf = sibling.leaf
# give custody of sibling's children to parent
sibling.left.parent = sibling.right.parent = parent
return parent


def merge(*iterables, key=None, reverse=False):
'''Merge multiple sorted inputs into a single sorted output.

Expand All @@ -329,67 +428,50 @@ def merge(*iterables, key=None, reverse=False):

'''

h = []
h_append = h.append

if reverse:
_heapify = _heapify_max
_heappop = _heappop_max
_heapreplace = _heapreplace_max
direction = -1
else:
_heapify = heapify
_heappop = heappop
_heapreplace = heapreplace
direction = 1

if key is None:
for order, it in enumerate(map(iter, iterables)):
try:
next = it.__next__
h_append([next(), order * direction, next])
except StopIteration:
pass
_heapify(h)
while len(h) > 1:
try:
while True:
value, order, next = s = h[0]
yield value
s[0] = next() # raises StopIteration when exhausted
_heapreplace(h, s) # restore heap condition
except StopIteration:
_heappop(h) # remove empty iterator
if h:
# fast case when only a single iterator remains
value, order, next = h[0]
yield value
yield from next.__self__
root = _MergeNode.build_tree(iterables, key, reverse)
if root is None:
return
elif root.leaf is root:
yield root.left
yield from root.right
return

for order, it in enumerate(map(iter, iterables)):
try:
next = it.__next__
value = next()
h_append([key(value), order * direction, value, next])
except StopIteration:
pass
_heapify(h)
while len(h) > 1:
_next, _StopIteration = next, StopIteration
while True:
# To find the value to yield, check which leaf
# the root's key came from.
node = root.leaf
yield node.left

# refill the leaf with one value from the iterator.
try:
while True:
key_value, order, value, next = s = h[0]
yield value
value = next()
s[0] = key(value)
s[2] = value
_heapreplace(h, s)
except StopIteration:
_heappop(h)
if h:
key_value, order, value, next = h[0]
yield value
yield from next.__self__
node.left = val = _next(node.right)
except _StopIteration:
node = node.promote_sibling()
if root.leaf is root:
yield root.left
yield from root.right
return
else:
node.key = val if key is None else key(val)

# The tight loop: For each ancestor of the chosen leaf,
# re-evaluate which of its children is higher priority,
# then take that winning child's key and leaf references.
if reverse:
while node is not root:
node = node.parent
left, right = node.left, node.right
winner = right if left.key < right.key else left
node.key = winner.key
node.leaf = winner.leaf
else:
while node is not root:
node = node.parent
left, right = node.left, node.right
winner = right if right.key < left.key else left
node.key = winner.key
node.leaf = winner.leaf


# Algorithm notes for nlargest() and nsmallest()
Expand Down
93 changes: 75 additions & 18 deletions Lib/test/test_heapq.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@
from test import support
from unittest import TestCase, skipUnless
from operator import itemgetter
from itertools import chain, product

py_heapq = support.import_fresh_module('heapq', blocked=['_heapq'])
c_heapq = support.import_fresh_module('heapq', fresh=['_heapq'])

# _heapq.nlargest/nsmallest are saved in heapq._nlargest/_smallest when
# _heapq is imported, so check them there
func_names = ['heapify', 'heappop', 'heappush', 'heappushpop', 'heapreplace',
'_heappop_max', '_heapreplace_max', '_heapify_max']
'_heappop_max', '_heapreplace_max', '_heapify_max', 'merge']

class TestModules(TestCase):
def test_py_functions(self):
Expand Down Expand Up @@ -176,22 +177,23 @@ def test_heapsort(self):
self.assertEqual(heap_sorted, sorted(data))

def test_merge(self):
inputs = []
for i in range(random.randrange(25)):
row = []
for j in range(random.randrange(100)):
tup = random.choice('ABC'), random.randrange(-500, 500)
row.append(tup)
inputs.append(row)

for key in [None, itemgetter(0), itemgetter(1), itemgetter(1, 0)]:
for reverse in [False, True]:
seqs = []
for seq in inputs:
seqs.append(sorted(seq, key=key, reverse=reverse))
self.assertEqual(sorted(chain(*inputs), key=key, reverse=reverse),
list(self.module.merge(*seqs, key=key, reverse=reverse)))
self.assertEqual(list(self.module.merge()), [])
lengths = range(25)
keys = [None, itemgetter(0), itemgetter(1), itemgetter(1, 0)]
reverses = [False, True]
for n, key, reverse in product(lengths, keys, reverses):
inputs = [
sorted(
[(random.choice('ABC'), random.randrange(-500, 500))
for _ in range(random.randrange(20))],
key=key, reverse=reverse
)
for _ in range(n)
]
expected = sorted(chain(*inputs), key=key, reverse=reverse)
with self.subTest(inputs=inputs, key=key, reverse=reverse):
m = self.module.merge(*inputs, key=key, reverse=reverse)
self.assertEqual(list(m), expected)
self.assertEqual(list(m), [])

def test_empty_merges(self):
# Merging two empty lists (with or without a key) should produce
Expand Down Expand Up @@ -358,7 +360,6 @@ def __iter__(self):
def __next__(self):
raise StopIteration

from itertools import chain
def L(seqn):
'Test multiple tiers of iterators'
return chain(map(lambda x:x, R(Ig(G(seqn)))))
Expand Down Expand Up @@ -464,6 +465,62 @@ def __lt__(self, o):
self.assertRaises((IndexError, RuntimeError), self.module.heappush, list1, g(1))
self.assertRaises((IndexError, RuntimeError), self.module.heappush, list2, h(1))

def test_merge_err(self):
nexterr_immediate = E
def nexterr_delayed():
yield from range(10)
raise ZeroDivisionError

def get_object_that_stops_iterating(length):
class StopsBeingIterable:
def __init__(self):
self.state = length
def __iter__(self):
return self
def __next__(self):
self.state -= 1
if self.state <= 0:
del type(self).__next__
return 42
return StopsBeingIterable()

merge = self.module.merge
for key in [None, lambda x:x]:
for reverse in [False, True]:
# test comparison failure during initialization
args = [[CmpErr()]] + [range(100)]*10
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)

for n in range(2,10):
# test comparison failure during intermediate steps
args = [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, CmpErr()]]*n
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
for n in range(1, 10):
# test __next__ error during initialization
args = [nexterr_immediate(range(10)) for _ in range(n)]
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
for n in range(1, 10):
# test __next__ error during intermediate steps
args = [nexterr_delayed() for _ in range(n)]
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
for n in range(1, 10):
for length in [0, 1, 2, 3, 4, 5, 10, 100]:
args = [get_object_that_stops_iterating(length)
for _ in range(n)]
mo = merge(*args, key=key, reverse=reverse)
self.assertRaises((TypeError, AttributeError), list, mo)

for reverse in [False, True]:
# test error during key computation
mo = merge(range(10), range(10), key=lambda x: x // 0, reverse=reverse)
self.assertRaises(ZeroDivisionError, list, mo)
mo = merge(range(10), key=object(), reverse=reverse)
self.assertRaises(TypeError, list, merge)

class TestErrorHandlingPython(TestErrorHandling, TestCase):
module = py_heapq

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Use a more optimized :func:`heapq.merge()` algorithm and add a C implementation. Patch by Dennis Sweeney.
Loading