Skip to content

Commit a886922

Browse files
ShaneHarveyShaneHarvey
committed
PYTHON-1684 Support sharded transactions recovery token (mongodb#406)
Transient errors inside transaction unpins the session. Add brief docs about sharded transactions and add 3.9 changelog. Tests changes: Add spec tests for sharded transaction recoveryToken. Speed up txn tests by reducing SDAM waiting time after a network error. Remove outdated test workaround for killAllSessions.
1 parent 0556578 commit a886922

34 files changed

+6015
-73
lines changed

‎doc/changelog.rst‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,14 @@
11
Changelog
22
=========
33

4-
Changes in Version 3.8.0
5-
------------------------
6-
7-
.. warning:: PyMongo no longer supports Python 2.6. RHEL 6 users should install
8-
Python 2.7 or newer from `Red Hat Software Collections
9-
<https://www.softwarecollections.org>`_. CentOS 6 users should install Python
10-
2.7 or newer from `SCL
11-
<https://wiki.centos.org/AdditionalResources/Repositories/SCL>`_
12-
13-
.. warning:: PyMongo no longer supports PyPy3 versions older than 3.5. Users
14-
must upgrade to PyPy3.5+.
15-
16-
- :class:`~bson.objectid.ObjectId` now implements the `ObjectID specification
17-
version 0.2 <https://github.com/mongodb/specifications/blob/master/source/objectid.rst>`_.
4+
Changes in Version 3.9.0.dev0
5+
-----------------------------
186

7+
Version 3.9 adds support for MongoDB 4.2. Highlights include:
198

20-
- Version 3.8.0 implements the `URI options specification`_ in the
9+
- Support for MongoDB 4.2 sharded transactions. Sharded transactions have
10+
the same API as replica set transactions. See :ref:`transactions-ref`.
11+
- Implement the `URI options specification`_ in the
2112
:meth:`~pymongo.mongo_client.MongoClient` constructor. Consequently, there are
2213
a number of changes in connection options:
2314

@@ -38,8 +29,31 @@ Changes in Version 3.8.0
3829
- ``ssl_pem_passphrase`` has been deprecated in favor of ``tlsCertificateKeyFilePassword``.
3930

4031

41-
.. _URI options specification: https://github.com/mongodb/specifications/blob/master/source/uri-options/uri-options.rst`
32+
.. _URI options specification: https://github.com/mongodb/specifications/blob/master/source/uri-options/uri-options.rst
33+
34+
35+
Issues Resolved
36+
...............
37+
38+
See the `PyMongo 3.9 release notes in JIRA`_ for the list of resolved issues
39+
in this release.
40+
41+
.. _PyMongo 3.9 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=21787
42+
43+
Changes in Version 3.8.0.dev0
44+
-----------------------------
45+
46+
.. warning:: PyMongo no longer supports Python 2.6. RHEL 6 users should install
47+
Python 2.7 or newer from `Red Hat Software Collections
48+
<https://www.softwarecollections.org>`_. CentOS 6 users should install Python
49+
2.7 or newer from `SCL
50+
<https://wiki.centos.org/AdditionalResources/Repositories/SCL>`_
51+
52+
.. warning:: PyMongo no longer supports PyPy3 versions older than 3.5. Users
53+
must upgrade to PyPy3.5+.
4254

55+
- :class:`~bson.objectid.ObjectId` now implements the `ObjectID specification
56+
version 0.2 <https://github.com/mongodb/specifications/blob/master/source/objectid.rst>`_.
4357

4458
Issues Resolved
4559
...............

‎pymongo/bulk.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ def _execute_command(self, generator, write_concern, session,
292292
if not to_send:
293293
raise InvalidOperation("cannot do an empty bulk write")
294294
result = bwc.write_command(request_id, msg, to_send)
295-
client._receive_cluster_time(result, session)
295+
client._process_response(result, session)
296296

297297
# Retryable writeConcernErrors halt the execution of this run.
298298
wce = result.get('writeConcernError', {})

‎pymongo/client_session.py‎

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,19 @@
7575
7676
.. versionadded:: 3.7
7777
78+
Sharded Transactions
79+
^^^^^^^^^^^^^^^^^^^^
80+
81+
PyMongo 3.9 adds support for transactions on sharded clusters running MongoDB
82+
4.2. Sharded transactions have the same API as replica set transactions.
83+
When running a transaction against a sharded cluster, the session is
84+
pinned to the mongos server selected for the first operation in the
85+
transaction. All subsequent operations that are part of the same transaction
86+
are routed to the same mongos server. When the transaction is completed, by
87+
running either commitTransaction or abortTransaction, the session is unpinned.
88+
89+
.. versionadded:: 3.9
90+
7891
.. mongodoc:: transactions
7992
8093
Classes
@@ -88,6 +101,7 @@
88101
from bson.binary import Binary
89102
from bson.int64 import Int64
90103
from bson.py3compat import abc, reraise_instance
104+
from bson.son import SON
91105
from bson.timestamp import Timestamp
92106

93107
from pymongo import monotonic
@@ -245,12 +259,19 @@ class _Transaction(object):
245259
def __init__(self, opts):
246260
self.opts = opts
247261
self.state = _TxnState.NONE
248-
self.transaction_id = 0
262+
self.sharded = False
249263
self.pinned_address = None
264+
self.recovery_token = None
250265

251266
def active(self):
252267
return self.state in (_TxnState.STARTING, _TxnState.IN_PROGRESS)
253268

269+
def reset(self):
270+
self.state = _TxnState.NONE
271+
self.sharded = False
272+
self.pinned_address = None
273+
self.recovery_token = None
274+
254275

255276
def _reraise_with_unknown_commit(exc):
256277
"""Re-raise an exception with the UnknownTransactionCommitResult label."""
@@ -367,10 +388,9 @@ def start_transaction(self, read_concern=None, write_concern=None,
367388

368389
self._transaction.opts = TransactionOptions(
369390
read_concern, write_concern, read_preference)
391+
self._transaction.reset()
370392
self._transaction.state = _TxnState.STARTING
371393
self._start_retryable_write()
372-
self._transaction.transaction_id = self._server_session.transaction_id
373-
self._transaction.pinned_address = None
374394
return _TransactionContext(self)
375395

376396
def commit_transaction(self):
@@ -482,15 +502,19 @@ def _finish_transaction(self, command_name, retrying):
482502
# subsequent commitTransaction commands should be upgraded to use
483503
# w:"majority" and set a default value of 10 seconds for wtimeout.
484504
wc = self._transaction.opts.write_concern
485-
if retrying and command_name == "commitTransaction":
505+
is_commit = command_name == "commitTransaction"
506+
if retrying and is_commit:
486507
wc_doc = wc.document
487508
wc_doc["w"] = "majority"
488509
wc_doc.setdefault("wtimeout", 10000)
489510
wc = WriteConcern(**wc_doc)
511+
cmd = SON([(command_name, 1)])
512+
if self._transaction.recovery_token and is_commit:
513+
cmd['recoveryToken'] = self._transaction.recovery_token
490514
with self._client._socket_for_writes(self) as sock_info:
491515
return self._client.admin._command(
492516
sock_info,
493-
command_name,
517+
cmd,
494518
session=self,
495519
write_concern=wc,
496520
parse_write_concern_error=True)
@@ -539,6 +563,15 @@ def advance_operation_time(self, operation_time):
539563
"of bson.timestamp.Timestamp")
540564
self._advance_operation_time(operation_time)
541565

566+
def _process_response(self, reply):
567+
"""Process a response to a command that was run with this session."""
568+
self._advance_cluster_time(reply.get('$clusterTime'))
569+
self._advance_operation_time(reply.get('operationTime'))
570+
if self._in_transaction and self._transaction.sharded:
571+
recovery_token = reply.get('recoveryToken')
572+
if recovery_token:
573+
self._transaction.recovery_token = recovery_token
574+
542575
@property
543576
def has_ended(self):
544577
"""True if this session is finished."""
@@ -558,8 +591,13 @@ def _pinned_address(self):
558591

559592
def _pin_mongos(self, server):
560593
"""Pin this session to the given mongos Server."""
594+
self._transaction.sharded = True
561595
self._transaction.pinned_address = server.description.address
562596

597+
def _unpin_mongos(self):
598+
"""Unpin this session from any pinned mongos address."""
599+
self._transaction.pinned_address = None
600+
563601
def _txn_read_preference(self):
564602
"""Return read preference of this transaction or None."""
565603
if self._in_transaction:
@@ -573,8 +611,7 @@ def _apply_to(self, command, is_retryable, read_preference):
573611
command['lsid'] = self._server_session.session_id
574612

575613
if not self._in_transaction:
576-
self._transaction.state = _TxnState.NONE
577-
self._transaction.pinned_address = None
614+
self._transaction.reset()
578615

579616
if is_retryable:
580617
command['txnNumber'] = self._server_session.transaction_id

‎pymongo/command_cursor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,13 @@ def duration(): return datetime.datetime.now() - start
149149
reply = response.data
150150

151151
try:
152-
with client._reset_on_error(self.__address):
152+
with client._reset_on_error(self.__address, self.__session):
153153
docs = self._unpack_response(reply,
154154
self.__id,
155155
self.__collection.codec_options)
156156
if from_command:
157157
first = docs[0]
158-
client._receive_cluster_time(first, self.__session)
158+
client._process_response(first, self.__session)
159159
helpers._check_command_response(first)
160160

161161
except OperationFailure as exc:

‎pymongo/cursor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -973,13 +973,13 @@ def duration(): return datetime.datetime.now() - start
973973
raise
974974

975975
try:
976-
with client._reset_on_error(self.__address):
976+
with client._reset_on_error(self.__address, self.__session):
977977
docs = self._unpack_response(reply,
978978
self.__id,
979979
self.__collection.codec_options)
980980
if from_command:
981981
first = docs[0]
982-
client._receive_cluster_time(first, self.__session)
982+
client._process_response(first, self.__session)
983983
helpers._check_command_response(first)
984984
except OperationFailure as exc:
985985
self.__killed = True

‎pymongo/mongo_client.py‎

Lines changed: 39 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1112,8 +1112,8 @@ def _get_topology(self):
11121112
return self._topology
11131113

11141114
@contextlib.contextmanager
1115-
def _get_socket(self, server):
1116-
with self._reset_on_error(server.description.address):
1115+
def _get_socket(self, server, session):
1116+
with self._reset_on_error(server.description.address, session):
11171117
with server.get_socket(self.__all_credentials) as sock_info:
11181118
yield sock_info
11191119

@@ -1128,26 +1128,31 @@ def _select_server(self, server_selector, session, address=None):
11281128
- `address` (optional): Address when sending a message
11291129
to a specific server, used for getMore.
11301130
"""
1131-
topology = self._get_topology()
1132-
address = address or (session and session._pinned_address)
1133-
if address:
1134-
# We're running a getMore or this session is pinned to a mongos.
1135-
server = topology.select_server_by_address(address)
1136-
if not server:
1137-
raise AutoReconnect('server %s:%d no longer available'
1138-
% address)
1139-
else:
1140-
server = topology.select_server(server_selector)
1141-
# Pin this session to the selected server if it's performing a
1142-
# sharded transaction.
1143-
if server.description.mongos and (session and
1144-
session._in_transaction):
1145-
session._pin_mongos(server)
1146-
return server
1131+
try:
1132+
topology = self._get_topology()
1133+
address = address or (session and session._pinned_address)
1134+
if address:
1135+
# We're running a getMore or this session is pinned to a mongos.
1136+
server = topology.select_server_by_address(address)
1137+
if not server:
1138+
raise AutoReconnect('server %s:%d no longer available'
1139+
% address)
1140+
else:
1141+
server = topology.select_server(server_selector)
1142+
# Pin this session to the selected server if it's performing a
1143+
# sharded transaction.
1144+
if server.description.mongos and (session and
1145+
session._in_transaction):
1146+
session._pin_mongos(server)
1147+
return server
1148+
except PyMongoError as exc:
1149+
if session and exc.has_error_label("TransientTransactionError"):
1150+
session._unpin_mongos()
1151+
raise
11471152

11481153
def _socket_for_writes(self, session):
11491154
server = self._select_server(writable_server_selector, session)
1150-
return self._get_socket(server)
1155+
return self._get_socket(server, session)
11511156

11521157
@contextlib.contextmanager
11531158
def _socket_for_reads(self, read_preference, session):
@@ -1162,7 +1167,7 @@ def _socket_for_reads(self, read_preference, session):
11621167
single = topology.description.topology_type == TOPOLOGY_TYPE.Single
11631168
server = self._select_server(read_preference, session)
11641169

1165-
with self._get_socket(server) as sock_info:
1170+
with self._get_socket(server, session) as sock_info:
11661171
slave_ok = (single and not sock_info.is_mongos) or (
11671172
read_preference != ReadPreference.PRIMARY)
11681173
yield sock_info, slave_ok
@@ -1194,7 +1199,8 @@ def _send_message_with_response(self, operation, exhaust=False,
11941199
and server.description.server_type != SERVER_TYPE.Mongos) or (
11951200
operation.read_preference != ReadPreference.PRIMARY)
11961201

1197-
with self._reset_on_error(server.description.address):
1202+
with self._reset_on_error(server.description.address,
1203+
operation.session):
11981204
return server.send_message_with_response(
11991205
operation,
12001206
set_slave_ok,
@@ -1203,14 +1209,20 @@ def _send_message_with_response(self, operation, exhaust=False,
12031209
exhaust)
12041210

12051211
@contextlib.contextmanager
1206-
def _reset_on_error(self, server_address):
1212+
def _reset_on_error(self, server_address, session):
12071213
"""On "not master" or "node is recovering" errors reset the server
12081214
according to the SDAM spec.
12091215
12101216
Unpin the session on transient transaction errors.
12111217
"""
12121218
try:
1213-
yield
1219+
try:
1220+
yield
1221+
except PyMongoError as exc:
1222+
if session and exc.has_error_label(
1223+
"TransientTransactionError"):
1224+
session._unpin_mongos()
1225+
raise
12141226
except NetworkTimeout:
12151227
# The socket has been closed. Don't reset the server.
12161228
# Server Discovery And Monitoring Spec: "When an application
@@ -1264,7 +1276,7 @@ def is_retrying():
12641276
supports_session = (
12651277
session is not None and
12661278
server.description.retryable_writes_supported)
1267-
with self._get_socket(server) as sock_info:
1279+
with self._get_socket(server, session) as sock_info:
12681280
if retryable and not supports_session:
12691281
if is_retrying():
12701282
# A retry is not possible because this server does
@@ -1674,12 +1686,10 @@ def _send_cluster_time(self, command, session):
16741686
if cluster_time:
16751687
command['$clusterTime'] = cluster_time
16761688

1677-
def _receive_cluster_time(self, reply, session):
1678-
cluster_time = reply.get('$clusterTime')
1679-
self._topology.receive_cluster_time(cluster_time)
1689+
def _process_response(self, reply, session):
1690+
self._topology.receive_cluster_time(reply.get('$clusterTime'))
16801691
if session is not None:
1681-
session._advance_cluster_time(cluster_time)
1682-
session._advance_operation_time(reply.get("operationTime"))
1692+
session._process_response(reply)
16831693

16841694
def server_info(self, session=None):
16851695
"""Get information about the MongoDB server we're connected to.

‎pymongo/network.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def command(sock, dbname, spec, slave_ok, is_mongos,
143143

144144
response_doc = unpacked_docs[0]
145145
if client:
146-
client._receive_cluster_time(response_doc, session)
146+
client._process_response(response_doc, session)
147147
if check:
148148
helpers._check_command_response(
149149
response_doc, None, allowable_errors,

0 commit comments

Comments
 (0)