Skip to content

Commit 703ecb2

Browse files
bdracobdraco
authored
feat: speed up question and answer history with a cython pxd (#1234)
1 parent 84054ce commit 703ecb2

8 files changed

Lines changed: 63 additions & 25 deletions

File tree

‎build_ext.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def build(setup_kwargs: Any) -> None:
2525
[
2626
"src/zeroconf/_dns.py",
2727
"src/zeroconf/_cache.py",
28+
"src/zeroconf/_history.py",
2829
"src/zeroconf/_listener.py",
2930
"src/zeroconf/_protocol/incoming.py",
3031
"src/zeroconf/_protocol/outgoing.py",

‎src/zeroconf/_history.pxd‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import cython
2+
3+
4+
cdef cython.double _DUPLICATE_QUESTION_INTERVAL
5+
6+
cdef class QuestionHistory:
7+
8+
cdef cython.dict _history
9+
10+
11+
@cython.locals(than=cython.double, previous_question=cython.tuple, previous_known_answers=cython.set)
12+
cpdef suppresses(self, object question, cython.double now, cython.set known_answers)
13+
14+
15+
@cython.locals(than=cython.double, now_known_answers=cython.tuple)
16+
cpdef async_expire(self, cython.double now)

‎src/zeroconf/_history.py‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,29 @@
2020
USA
2121
"""
2222

23-
from typing import Dict, Set, Tuple
23+
from typing import Dict, List, Set, Tuple
2424

2525
from ._dns import DNSQuestion, DNSRecord
2626
from .const import _DUPLICATE_QUESTION_INTERVAL
2727

2828
# The QuestionHistory is used to implement Duplicate Question Suppression
2929
# https://datatracker.ietf.org/doc/html/rfc6762#section-7.3
3030

31+
_float = float
32+
3133

3234
class QuestionHistory:
35+
"""Remember questions and known answers."""
36+
3337
def __init__(self) -> None:
38+
"""Init a new QuestionHistory."""
3439
self._history: Dict[DNSQuestion, Tuple[float, Set[DNSRecord]]] = {}
3540

36-
def add_question_at_time(self, question: DNSQuestion, now: float, known_answers: Set[DNSRecord]) -> None:
41+
def add_question_at_time(self, question: DNSQuestion, now: _float, known_answers: Set[DNSRecord]) -> None:
3742
"""Remember a question with known answers."""
3843
self._history[question] = (now, known_answers)
3944

40-
def suppresses(self, question: DNSQuestion, now: float, known_answers: Set[DNSRecord]) -> bool:
45+
def suppresses(self, question: DNSQuestion, now: _float, known_answers: Set[DNSRecord]) -> bool:
4146
"""Check to see if a question should be suppressed.
4247
4348
https://datatracker.ietf.org/doc/html/rfc6762#section-7.3
@@ -59,12 +64,16 @@ def suppresses(self, question: DNSQuestion, now: float, known_answers: Set[DNSRe
5964
return False
6065
return True
6166

62-
def async_expire(self, now: float) -> None:
67+
def async_expire(self, now: _float) -> None:
6368
"""Expire the history of old questions."""
64-
removes = [
65-
question
66-
for question, now_known_answers in self._history.items()
67-
if now - now_known_answers[0] > _DUPLICATE_QUESTION_INTERVAL
68-
]
69+
removes: List[DNSQuestion] = []
70+
for question, now_known_answers in self._history.items():
71+
than, _ = now_known_answers
72+
if now - than > _DUPLICATE_QUESTION_INTERVAL:
73+
removes.append(question)
6974
for question in removes:
7075
del self._history[question]
76+
77+
def clear(self) -> None:
78+
"""Clear the history."""
79+
self._history.clear()

‎tests/__init__.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,17 @@
2323
import asyncio
2424
import socket
2525
from functools import lru_cache
26-
from typing import List
26+
from typing import List, Set
2727

2828
import ifaddr
2929

30-
from zeroconf import DNSIncoming, Zeroconf
30+
from zeroconf import DNSIncoming, DNSQuestion, DNSRecord, Zeroconf
31+
from zeroconf._history import QuestionHistory
32+
33+
34+
class QuestionHistoryWithoutSuppression(QuestionHistory):
35+
def suppresses(self, question: DNSQuestion, now: float, known_answers: Set[DNSRecord]) -> bool:
36+
return False
3137

3238

3339
def _inject_responses(zc: Zeroconf, msgs: List[DNSIncoming]) -> None:
@@ -77,4 +83,4 @@ def has_working_ipv6():
7783

7884
def _clear_cache(zc):
7985
zc.cache.cache.clear()
80-
zc.question_history._history.clear()
86+
zc.question_history.clear()

‎tests/services/test_browser.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,12 @@
3131
from zeroconf._services.info import ServiceInfo
3232
from zeroconf.asyncio import AsyncZeroconf
3333

34-
from .. import _inject_response, _wait_for_start, has_working_ipv6
34+
from .. import (
35+
QuestionHistoryWithoutSuppression,
36+
_inject_response,
37+
_wait_for_start,
38+
has_working_ipv6,
39+
)
3540

3641
log = logging.getLogger('zeroconf')
3742
original_logging_level = logging.NOTSET
@@ -444,6 +449,7 @@ def test_backoff():
444449
type_ = "_http._tcp.local."
445450
zeroconf_browser = Zeroconf(interfaces=['127.0.0.1'])
446451
_wait_for_start(zeroconf_browser)
452+
zeroconf_browser.question_history = QuestionHistoryWithoutSuppression()
447453

448454
# we are going to patch the zeroconf send to check query transmission
449455
old_send = zeroconf_browser.async_send
@@ -465,10 +471,8 @@ def send(out, addr=const._MDNS_ADDR, port=const._MDNS_PORT, v6_flow_scope=()):
465471
# patch the zeroconf current_time_millis
466472
# patch the backoff limit to prevent test running forever
467473
with patch.object(zeroconf_browser, "async_send", send), patch.object(
468-
zeroconf_browser.question_history, "suppresses", return_value=False
469-
), patch.object(_services_browser, "current_time_millis", current_time_millis), patch.object(
470-
_services_browser, "_BROWSER_BACKOFF_LIMIT", 10
471-
), patch.object(
474+
_services_browser, "current_time_millis", current_time_millis
475+
), patch.object(_services_browser, "_BROWSER_BACKOFF_LIMIT", 10), patch.object(
472476
_services_browser, "_FIRST_QUERY_DELAY_RANDOM_INTERVAL", (0, 0)
473477
):
474478
# dummy service callback

‎tests/test_asyncio.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
)
4444
from zeroconf.const import _LISTENER_TIME
4545

46-
from . import _clear_cache, has_working_ipv6
46+
from . import QuestionHistoryWithoutSuppression, _clear_cache, has_working_ipv6
4747

4848
log = logging.getLogger('zeroconf')
4949
original_logging_level = logging.NOTSET
@@ -951,6 +951,7 @@ def on_service_state_change(zeroconf, service_type, state_change, name):
951951

952952
aiozc = AsyncZeroconf(interfaces=['127.0.0.1'])
953953
zeroconf_browser = aiozc.zeroconf
954+
zeroconf_browser.question_history = QuestionHistoryWithoutSuppression()
954955
await zeroconf_browser.async_wait_for_start()
955956

956957
# we are going to patch the zeroconf send to check packet sizes
@@ -990,11 +991,9 @@ def send(out, addr=const._MDNS_ADDR, port=const._MDNS_PORT, v6_flow_scope=()):
990991
# patch the backoff limit to ensure we always get one query every 1/4 of the DNS TTL
991992
# Disable duplicate question suppression and duplicate packet suppression for this test as it works
992993
# by asking the same question over and over
993-
with patch.object(zeroconf_browser.question_history, "suppresses", return_value=False), patch.object(
994-
zeroconf_browser, "async_send", send
995-
), patch("zeroconf._services.browser.current_time_millis", _new_current_time_millis), patch.object(
996-
_services_browser, "_BROWSER_BACKOFF_LIMIT", int(expected_ttl / 4)
997-
):
994+
with patch.object(zeroconf_browser, "async_send", send), patch(
995+
"zeroconf._services.browser.current_time_millis", _new_current_time_millis
996+
), patch.object(_services_browser, "_BROWSER_BACKOFF_LIMIT", int(expected_ttl / 4)):
998997
service_added = asyncio.Event()
999998
service_removed = asyncio.Event()
1000999

‎tests/test_handlers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1131,7 +1131,7 @@ async def test_cache_flush_bit():
11311131
for record in new_records:
11321132
assert zc.cache.async_get_unique(record) is not None
11331133

1134-
original_a_record.created = current_time_millis() - 1001
1134+
original_a_record.created = current_time_millis() - 1500
11351135

11361136
# Do the run within 1s to verify the original record is not going to be expired
11371137
out = r.DNSOutgoing(const._FLAGS_QR_RESPONSE | const._FLAGS_AA, multicast=True)
@@ -1146,7 +1146,7 @@ async def test_cache_flush_bit():
11461146
cached_records = [zc.cache.async_get_unique(record) for record in new_records]
11471147
for cached_record in cached_records:
11481148
assert cached_record is not None
1149-
cached_record.created = current_time_millis() - 1001
1149+
cached_record.created = current_time_millis() - 1500
11501150

11511151
fresh_address = socket.inet_aton("4.4.4.4")
11521152
info.addresses = [fresh_address]

‎tests/test_listener.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
from zeroconf._protocol import outgoing
1515
from zeroconf._protocol.incoming import DNSIncoming
1616

17+
from . import QuestionHistoryWithoutSuppression
18+
1719
log = logging.getLogger('zeroconf')
1820
original_logging_level = logging.NOTSET
1921

@@ -123,6 +125,7 @@ def test_guard_against_duplicate_packets():
123125
These packets can quickly overwhelm the system.
124126
"""
125127
zc = Zeroconf(interfaces=['127.0.0.1'])
128+
zc.question_history = QuestionHistoryWithoutSuppression()
126129

127130
class SubListener(_listener.AsyncListener):
128131
def handle_query_or_defer(

0 commit comments

Comments
 (0)