1818"""The driver: split a query into tasks, fan them out, collect the answer.
1919
2020The shape is deliberately boring, because the interesting part is not the
21- scheduling. What matters is the four things the driver has to get right, each
21+ scheduling. What matters is the five things the driver has to get right, each
2222of which is a way a real deployment goes wrong:
2323
24- 1. It serializes the stage **with** its session. ``to_bytes(None)`` uses an
24+ 1. It serializes each stage **with** its session. ``to_bytes(None)`` uses an
2525 empty codec chain and cannot encode any library's node.
26262. It ships the codec ids it used, so a worker can refuse a plan it would
2727 misread rather than decode it with the wrong codec.
3030 cannot disagree.
31314. It waits for every worker before reading, because the stage node decides
3232 whether to read or recompute by looking at the filesystem.
33+ 5. It ships *every* stage. A plan can hold more than one -- an aggregate in
34+ each branch of a union, say -- and they are independent subtrees rather
35+ than a chain.
3336"""
3437
3538from __future__ import annotations
4952 from datafusion .plan import ExecutionPlan
5053 from datafusion .user_defined import ScalarUDF
5154
52- __all__ = ["DistributedResult" , "find_stage" , "run_distributed" ]
55+ __all__ = [
56+ "DistributedResult" ,
57+ "find_stage" ,
58+ "find_stages" ,
59+ "run_distributed" ,
60+ ]
5361
5462
5563class DistributedResult :
@@ -58,21 +66,26 @@ class DistributedResult:
5866 def __init__ (
5967 self ,
6068 batches : list [pa .RecordBatch ],
61- partitions : list [int ],
62- worker_rows : dict [int , int ],
69+ tasks : list [tuple [ int , int ] ],
70+ task_rows : dict [tuple [ int , int ] , int ],
6371 ) -> None :
6472 self .batches = batches
65- self .partitions = partitions
66- """Partition indices that were dispatched, one per worker."""
67- self .worker_rows = worker_rows
68- """Rows each worker produced, keyed by partition index."""
73+ self .tasks = tasks
74+ """``(stage_id, partition)`` pairs that were dispatched, one per worker.
75+
76+ Keyed by both, not by partition alone: a query with an aggregate in
77+ more than one branch has more than one stage, and partition 0 of each
78+ is a different piece of work.
79+ """
80+ self .task_rows = task_rows
81+ """Rows each worker produced, keyed as :attr:`tasks` is."""
6982
7083
7184STAGE_NODE_NAME = "ShuffleStageExec"
7285
7386
74- def find_stage (plan : ExecutionPlan ) -> ExecutionPlan | None :
75- """Locate the stage node the planner inserted.
87+ def find_stages (plan : ExecutionPlan ) -> list [ ExecutionPlan ] :
88+ """Locate every stage node the planner inserted, in pre-order .
7689
7790 Matched on the display string because a Python caller has no way to
7891 downcast a Rust plan node -- there is no ``isinstance`` across an FFI
@@ -87,27 +100,39 @@ def find_stage(plan: ExecutionPlan) -> ExecutionPlan | None:
87100 A foreign node reports its own name nested inside the wrapper's, which
88101 makes anchored matches on plan text quietly wrong -- the kind of thing
89102 that works in a single-library test and fails the moment a real extension
90- is involved.
103+ is involved. It is also why the stage *id* has to be recomputed here
104+ rather than read: the wrapper dropped it.
105+
106+ Pre-order is the contract with the planner, which numbers stages in the
107+ same walk, so the nth node returned here has the id
108+ ``_internal.stage_id(n)``. The recursion stops at a stage rather than
109+ descending into it, because the planner never nests one inside another.
91110 """
92111 if STAGE_NODE_NAME in plan .display ():
93- return plan
94- for child in plan .children ():
95- found = find_stage (child )
96- if found is not None :
97- return found
98- return None
112+ return [plan ]
113+ return [stage for child in plan .children () for stage in find_stages (child )]
114+
115+
116+ def find_stage (plan : ExecutionPlan ) -> ExecutionPlan | None :
117+ """The first stage in `plan`, or ``None``.
118+
119+ For tests and callers that only care whether the planner split at all.
120+ See :func:`find_stages`.
121+ """
122+ stages = find_stages (plan )
123+ return stages [0 ] if stages else None
99124
100125
101126def _dispatch (
102- envelope : dict , envelope_dir : pathlib .Path , partition : int
127+ envelope : dict , envelope_dir : pathlib .Path , stage_id : int , partition : int
103128) -> subprocess .Popen [str ]:
104- """Start one worker for one partition.
129+ """Start one worker for one ``(stage, partition)`` .
105130
106131 ``sys.executable``, not ``python``: a worker on a different Python minor
107132 version cannot load a cloudpickled inline UDF, and that failure is far
108133 from its cause.
109134 """
110- path = envelope_dir / f"task-{ partition } .json"
135+ path = envelope_dir / f"task-{ stage_id } - { partition } .json"
111136 path .write_text (json .dumps (envelope ))
112137 return subprocess .Popen ( # noqa: S603
113138 [sys .executable , "-m" , "dfx_engine.worker" , str (path )],
@@ -120,7 +145,7 @@ def _dispatch(
120145def run_distributed (
121146 sql : str , spec : SessionSpec , extra_udfs : list [ScalarUDF ] | None = None
122147) -> DistributedResult :
123- """Run `sql`, executing its leaf stage in one worker process per partition .
148+ """Run `sql`, executing each stage partition in its own worker process.
124149
125150 Requires ``spec.shuffle_dir``: without it the planner inserts no stage and
126151 there is nothing to distribute.
@@ -135,13 +160,13 @@ def run_distributed(
135160 message = "run_distributed needs a shuffle_dir; build_session got none"
136161 raise ValueError (message )
137162
138- ctx , engine , _storage = build_session (spec )
163+ ctx , _engine , _storage = build_session (spec )
139164 for function in extra_udfs or []:
140165 ctx .register_udf (function )
141166 plan = ctx .sql (sql ).execution_plan ()
142167
143- stage = find_stage (plan )
144- if stage is None :
168+ stages = find_stages (plan )
169+ if not stages :
145170 message = (
146171 "no ShuffleStageExec in the plan; the engine's planner did not run, "
147172 "or its config extension was not registered"
@@ -151,48 +176,56 @@ def run_distributed(
151176 shuffle_dir = pathlib .Path (spec .shuffle_dir )
152177 shuffle_dir .mkdir (parents = True , exist_ok = True )
153178
154- # Encode the stage subtree, through the session that owns the codecs.
155- plan_path = shuffle_dir / "stage.plan"
156- plan_path .write_bytes (stage .to_bytes (ctx ))
157-
158- stage_id = _internal .stage_id ()
159- partitions = list (range (stage .partition_count ))
160- envelopes = [
161- {
162- "spec" : spec .to_json (),
163- "plan" : str (plan_path ),
164- "stage_id" : stage_id ,
165- "partition" : partition ,
166- }
167- for partition in partitions
168- ]
169-
170- # One process per partition, all in flight together. This is the claim the
179+ # One task per (stage, partition). Every stage is shipped, not just the
180+ # first: a query with an aggregate in two branches has two independent
181+ # subtrees, and leaving one behind would have the driver compute it
182+ # locally while the plan it shipped claimed otherwise.
183+ tasks : list [tuple [int , int ]] = []
184+ envelopes = []
185+ for index , stage in enumerate (stages ):
186+ # The id the planner gave this stage, recomputed from its position
187+ # because the FFI wrapper's display does not carry it.
188+ stage_id = _internal .stage_id (index )
189+ # Encode the stage subtree, through the session that owns the codecs.
190+ plan_path = shuffle_dir / f"stage-{ stage_id } .plan"
191+ plan_path .write_bytes (stage .to_bytes (ctx ))
192+ for partition in range (stage .partition_count ):
193+ tasks .append ((stage_id , partition ))
194+ envelopes .append (
195+ {
196+ "spec" : spec .to_json (),
197+ "plan" : str (plan_path ),
198+ "stage_id" : stage_id ,
199+ "partition" : partition ,
200+ }
201+ )
202+
203+ # One process per task, all in flight together. This is the claim the
171204 # example is making: each worker reads a different file and writes a
172205 # different result, so they need no coordination beyond the directory.
173206 workers = [
174- _dispatch (envelope , shuffle_dir , partition )
175- for envelope , partition in zip (envelopes , partitions , strict = True )
207+ _dispatch (envelope , shuffle_dir , stage_id , partition )
208+ for envelope , ( stage_id , partition ) in zip (envelopes , tasks , strict = True )
176209 ]
177210
178- worker_rows : dict [int , int ] = {}
211+ task_rows : dict [tuple [ int , int ] , int ] = {}
179212 failures = []
180- for partition , worker in zip (partitions , workers , strict = True ):
213+ for task , worker in zip (tasks , workers , strict = True ):
181214 stdout , stderr = worker .communicate ()
182215 if worker .returncode != 0 :
183- failures .append (f"partition { partition } failed:\n { stderr } " )
216+ stage_id , partition = task
217+ failures .append (f"stage { stage_id } partition { partition } failed:\n { stderr } " )
184218 continue
185- worker_rows [ partition ] = json .loads (stdout )["rows" ]
219+ task_rows [ task ] = json .loads (stdout )["rows" ]
186220
187221 if failures :
188222 raise RuntimeError ("\n " .join (failures ))
189223
190224 # Now run the whole query here. Every stage partition has a file, so the
191- # stage node streams them instead of recomputing -- the driver does only
225+ # stage nodes stream them instead of recomputing -- the driver does only
192226 # the final merge.
193227 batches = ctx .sql (sql ).collect ()
194- _ = engine
195- return DistributedResult (batches , partitions , worker_rows )
228+ return DistributedResult (batches , tasks , task_rows )
196229
197230
198231def run_local (
0 commit comments