Repository navigation
Expand file tree
/
Copy pathresolver.py
More file actions
432 lines (356 loc) · 18.9 KB
/
Copy pathresolver.py
File metadata and controls
432 lines (356 loc) · 18.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
from __future__ import annotations
import itertools
import typing as t
from sqlglot import exp
from sqlglot.dialects.dialect import Dialect
from sqlglot.errors import OptimizeError
from sqlglot.helper import seq_get, SingleValuedMapping
from sqlglot.optimizer.scope import Scope
if t.TYPE_CHECKING:
from sqlglot.schema import Schema
from collections.abc import Sequence, Mapping
class Resolver:
"""
Helper for resolving columns.
This is a class so we can lazily load some things and easily share them across functions.
"""
def __init__(self, scope: Scope, schema: Schema, infer_schema: bool = True) -> None:
self.scope: Scope = scope
self.schema: Schema = schema
self.dialect: Dialect = schema.dialect or Dialect()
self._source_columns: dict[str, Sequence[str]] | None = None
self._unambiguous_columns: Mapping[str, str] | None = None
self._all_columns: set[str] | None = None
self._infer_schema: bool = infer_schema
self._get_source_columns_cache: dict[tuple[str, bool], Sequence[str]] = {}
self._column_type_from_scope_cache: dict[tuple[int, str], exp.DataType | None] = {}
def get_table(self, column: str | exp.Column) -> exp.Identifier | None:
"""
Get the table for a column name.
Args:
column: The column expression (or column name) to find the table for.
Returns:
The table name if it can be found/inferred.
"""
column_name = column if isinstance(column, str) else column.name
table_name = self._get_table_name_from_sources(column_name)
if not table_name and isinstance(column, exp.Column):
# Fall-back case: If we couldn't find the `table_name` from ALL of the sources,
# attempt to disambiguate the column based on other characteristics e.g if this column is in a join condition,
# we may be able to disambiguate based on the source order.
if join_context := self._get_column_join_context(column):
# In this case, the return value will be the join that _may_ be able to disambiguate the column
# and we can use the source columns available at that join to get the table name
# catch OptimizeError if column is still ambiguous and try to resolve with schema inference below
try:
table_name = self._get_table_name_from_sources(
column_name, self._get_available_source_columns(join_context)
)
except OptimizeError:
pass
if not table_name and self._infer_schema:
sources_without_schema = tuple(
source
for source, columns in self._get_all_source_columns().items()
if not columns or "*" in columns
)
if len(sources_without_schema) == 1:
table_name = sources_without_schema[0]
if table_name not in self.scope.selected_sources:
return exp.to_identifier(table_name)
node: exp.Expr = self.scope.selected_sources[table_name][0]
if isinstance(node, exp.Query):
while node and node.alias != table_name and node.parent:
node = node.parent
node_alias = node.args.get("alias")
if node_alias:
return exp.to_identifier(node_alias.this)
return exp.to_identifier(table_name)
@property
def all_columns(self) -> set[str]:
"""All available columns of all sources in this scope"""
if self._all_columns is None:
self._all_columns = {
column for columns in self._get_all_source_columns().values() for column in columns
}
return self._all_columns
def get_source_columns_from_set_op(self, expression: exp.Expr) -> list[str]:
if isinstance(expression, exp.Select):
return expression.named_selects
if isinstance(expression, exp.Subquery) and isinstance(expression.this, exp.SetOperation):
# Different types of SET modifiers can be chained together if they're explicitly grouped by nesting
return self.get_source_columns_from_set_op(expression.this)
if not isinstance(expression, exp.SetOperation):
raise OptimizeError(f"Unknown set operation: {expression}")
set_op = expression
# BigQuery specific set operations modifiers, e.g INNER UNION ALL BY NAME
on_column_list = set_op.args.get("on")
if on_column_list:
# The resulting columns are the columns in the ON clause:
# {INNER | LEFT | FULL} UNION ALL BY NAME ON (col1, col2, ...)
columns = [col.name for col in on_column_list]
elif set_op.side or set_op.kind:
side = set_op.side
kind = set_op.kind
# Visit the children UNIONs (if any) in a post-order traversal
left = self.get_source_columns_from_set_op(set_op.this)
right = self.get_source_columns_from_set_op(set_op.expression)
# We use dict.fromkeys to deduplicate keys and maintain insertion order
if side == "LEFT":
columns = left
elif side == "FULL":
columns = list(dict.fromkeys(left + right))
elif kind == "INNER":
columns = list(dict.fromkeys(left).keys() & dict.fromkeys(right).keys())
else:
columns = set_op.named_selects
return columns
def get_source_columns(self, name: str, only_visible: bool = False) -> Sequence[str]:
"""Resolve the source columns for a given source `name`."""
cache_key = (name, only_visible)
if cache_key not in self._get_source_columns_cache:
if name not in self.scope.sources:
raise OptimizeError(f"Unknown table: {name}")
source = self.scope.sources[name]
# A pivoted CTE reference is stored as an exp.Table in the scope sources (see
# _traverse_tables in scope.py), but the underlying CTE Scope still holds the
# column information we need to resolve pre-pivot columns.
if (
isinstance(source, exp.Table)
and not source.db
and source.args.get("pivots")
and source.name in self.scope.cte_sources
):
source = self.scope.cte_sources[source.name]
if isinstance(source, exp.Table):
columns = self.schema.column_names(source, only_visible)
elif isinstance(source, Scope) and isinstance(
source_expr := source.expression, (exp.Values, exp.Unnest, exp.Lateral)
):
columns = source_expr.named_selects
# in bigquery, unnest structs are automatically scoped as tables, so you can directly select
# a struct field in a query. This handles the case where the unnest is statically defined.
if self.dialect.UNNEST_COLUMN_ONLY and isinstance(source_expr, exp.Unnest):
if not source_expr.type or source_expr.type.is_type(exp.DType.UNKNOWN):
unnest_expr = seq_get(source_expr.expressions, 0)
if isinstance(unnest_expr, exp.Column) and self.scope.parent:
col_type = self._get_unnest_column_type(unnest_expr, self.scope.parent)
if col_type and col_type.is_type(exp.DType.ARRAY):
element_types = col_type.expressions
if element_types:
source_expr.type = element_types[0].copy()
elif col_type:
source_expr.type = col_type.copy()
columns.extend(self._struct_field_names(source_expr.type))
elif isinstance(source_expr, exp.Lateral) and isinstance(
source_expr.this, exp.Explode
):
explode_col = source_expr.this.this
# If the column is unqualified at this point, it couldn't be resolved when
# this scope's children were qualified; disambiguating it here would require
# enumerating this very source's columns, i.e recurse without bound
if isinstance(explode_col, exp.Column) and explode_col.table and source.parent:
col_type = self._get_unnest_column_type(explode_col, source.parent)
columns.extend(self._struct_field_names(col_type))
elif isinstance(source_expr, exp.Lateral) and isinstance(
source_expr.this, exp.Query
):
columns = source_expr.this.named_selects
elif isinstance(source, Scope) and isinstance(source.expression, exp.SetOperation):
columns = self.get_source_columns_from_set_op(source.expression)
else:
selectable = source.expression.assert_is(exp.Selectable)
select = seq_get(selectable.selects, 0)
if isinstance(select, exp.QueryTransform):
# https://spark.apache.org/docs/3.5.1/sql-ref-syntax-qry-select-transform.html
schema = select.args.get("schema")
columns = [c.name for c in schema.expressions] if schema else ["key", "value"]
else:
columns = selectable.named_selects
node, _ = self.scope.selected_sources.get(name) or (None, None)
if isinstance(node, Scope):
column_aliases = node.expression.alias_column_names
elif isinstance(node, exp.Expr):
column_aliases = node.alias_column_names
else:
column_aliases = []
if column_aliases:
# If the source's columns are aliased, their aliases shadow the corresponding column names.
# This can be expensive if there are lots of columns, so only do this if column_aliases exist.
columns = [
alias or name
for (name, alias) in itertools.zip_longest(columns, column_aliases)
]
self._get_source_columns_cache[cache_key] = columns
return self._get_source_columns_cache[cache_key]
def _get_all_source_columns(self) -> dict[str, Sequence[str]]:
if self._source_columns is None:
self._source_columns = {
source_name: self.get_source_columns(source_name)
for source_name, source in itertools.chain(
self.scope.selected_sources.items(), self.scope.lateral_sources.items()
)
}
return self._source_columns
def _get_table_name_from_sources(
self, column_name: str, source_columns: dict[str, Sequence[str]] | None = None
) -> str | None:
if not source_columns:
# If not supplied, get all sources to calculate unambiguous columns
if self._unambiguous_columns is None:
self._unambiguous_columns = self._get_unambiguous_columns(
self._get_all_source_columns()
)
unambiguous_columns = self._unambiguous_columns
else:
unambiguous_columns = self._get_unambiguous_columns(source_columns)
return unambiguous_columns.get(column_name)
def _get_column_join_context(self, column: exp.Column) -> exp.Join | None:
"""
Check if a column participating in a join can be qualified based on the source order.
"""
args = self.scope.expression.args
joins = args.get("joins")
if not joins or args.get("laterals") or args.get("pivots"):
# Feature gap: We currently don't try to disambiguate columns if other sources
# (e.g laterals, pivots) exist alongside joins
return None
join_ancestor = column.find_ancestor(exp.Join, exp.Select)
if (
isinstance(join_ancestor, exp.Join)
and join_ancestor.alias_or_name in self.scope.selected_sources
):
# Ensure that the found ancestor is a join that contains an actual source,
# e.g in Clickhouse `b` is an array expression in `a ARRAY JOIN b`
return join_ancestor
return None
def _get_available_source_columns(self, join_ancestor: exp.Join) -> dict[str, Sequence[str]]:
"""
Get the source columns that are available at the point where a column is referenced.
For columns in JOIN conditions, this only includes tables that have been joined
up to that point. Example:
```
SELECT * FROM t_1 INNER JOIN ... INNER JOIN t_n ON t_1.a = c INNER JOIN t_n+1 ON ...
``` ^
|
+----------------------------------+
|
⌄
The unqualified column `c` is not ambiguous if no other sources up until that
join i.e t_1, ..., t_n, contain a column named `c`.
"""
args = self.scope.expression.args
# Collect tables in order: FROM clause tables + joined tables up to current join
from_name = args["from_"].alias_or_name
available_sources = {from_name: self.get_source_columns(from_name)}
for join in args["joins"][: t.cast(int, join_ancestor.index) + 1]:
available_sources[join.alias_or_name] = self.get_source_columns(join.alias_or_name)
return available_sources
def _get_unambiguous_columns(
self, source_columns: dict[str, Sequence[str]]
) -> Mapping[str, str]:
"""
Find all the unambiguous columns in sources.
Args:
source_columns: Mapping of names to source columns.
Returns:
Mapping of column name to source name.
"""
if not source_columns:
return {}
source_columns_pairs = list(source_columns.items())
first_table, first_columns = source_columns_pairs[0]
if len(source_columns_pairs) == 1:
# Performance optimization - avoid copying first_columns if there is only one table.
return SingleValuedMapping(first_columns, first_table)
# For BigQuery UNNEST_COLUMN_ONLY, build a mapping of original UNNEST aliases
# from alias.columns[0] to their source names. This is used to resolve shadowing
# where an UNNEST alias shadows a column name from another table.
unnest_original_aliases: dict[str, str] = {}
if self.dialect.UNNEST_COLUMN_ONLY:
unnest_original_aliases = {
alias_arg.columns[0].name: source_name
for source_name, source in self.scope.sources.items()
if (
isinstance(source.expression, exp.Unnest)
and (alias_arg := source.expression.args.get("alias"))
and alias_arg.columns
)
}
unambiguous_columns = {col: first_table for col in first_columns}
all_columns = set(unambiguous_columns)
for table, columns in source_columns_pairs[1:]:
unique = set(columns)
ambiguous = all_columns.intersection(unique)
all_columns.update(columns)
for column in ambiguous:
if column in unnest_original_aliases:
unambiguous_columns[column] = unnest_original_aliases[column]
continue
unambiguous_columns.pop(column, None)
for column in unique.difference(ambiguous):
unambiguous_columns[column] = table
return unambiguous_columns
def _struct_field_names(self, col_type: exp.DataType | None) -> list[str]:
if col_type and col_type.is_type(exp.DType.ARRAY):
col_type = seq_get(col_type.expressions, 0)
return (
[k.name for k in col_type.expressions]
if col_type and col_type.is_type(exp.DType.STRUCT)
else []
)
def _get_unnest_column_type(self, column: exp.Column, scope: Scope) -> exp.DataType | None:
"""
Get the type of a column being unnested/exploded, tracing through CTEs/subqueries to find the base table.
Args:
column: The column expression being unnested/exploded.
scope: The scope to resolve the column in.
Returns:
The DataType of the column, or None if not found.
"""
# if column is qualified, use that table, otherwise disambiguate using the resolver
if column.table:
table_name = column.table
else:
# use the parent scope's resolver to disambiguate the column
parent_resolver = Resolver(scope, self.schema, self._infer_schema)
table_identifier = parent_resolver.get_table(column)
if not table_identifier:
return None
table_name = table_identifier.name
source = scope.sources.get(table_name)
return self._get_column_type_from_scope(source, column) if source else None
def _get_column_type_from_scope(
self, source: Scope | exp.Table, column: exp.Column
) -> exp.DataType | None:
"""
Get a column's type by tracing through scopes/tables to find the base table.
Args:
source: The source to search - can be a Scope (to iterate its sources) or a Table.
column: The column to find the type for.
Returns:
The DataType of the column, or None if not found.
"""
# A single source can be reachable through many paths of a scope DAG (e.g. a CTE
# referenced by several other CTEs). The schema and scope are immutable during
# qualification, so the type of `column` under `source` depends only on
# `(source, column name)`; memoize it to walk each source once.
cache_key = (id(source), column.name)
if cache_key in self._column_type_from_scope_cache:
return self._column_type_from_scope_cache[cache_key]
# None is a valid result if DataType could not be determined!
result: exp.DataType | None = None
if isinstance(source, exp.Table):
# base table - get the column type from schema
col_type = self.schema.get_column_type(source, column)
if col_type and not col_type.is_type(exp.DType.UNKNOWN):
result = col_type
elif isinstance(source, Scope):
# iterate over all sources in the scope
for nested_source in source.sources.values():
nested_type = self._get_column_type_from_scope(nested_source, column)
if nested_type and not nested_type.is_type(exp.DType.UNKNOWN):
result = nested_type
break
self._column_type_from_scope_cache[cache_key] = result
return result