-
Notifications
You must be signed in to change notification settings - Fork 9k
Expand file tree
/
Copy pathValidationUtils.cpp
More file actions
821 lines (694 loc) · 32.7 KB
/
Copy pathValidationUtils.cpp
File metadata and controls
821 lines (694 loc) · 32.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
#include <Analyzer/ValidationUtils.h>
#include <Interpreters/GetAggregatesVisitor.h>
#include <Analyzer/AggregationUtils.h>
#include <Analyzer/ArrayJoinNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/JoinNode.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/WindowFunctionsUtils.h>
#include <Analyzer/traverseQueryTree.h>
#include <Interpreters/Context.h>
#include <Interpreters/DatabaseCatalog.h>
#include <Storages/IStorage.h>
#include <Storages/StorageMaterializedView.h>
#include <Storages/StorageMerge.h>
#include <Storages/StorageProxy.h>
#include <memory>
#include <ranges>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int ILLEGAL_PREWHERE;
extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER;
extern const int LOGICAL_ERROR;
extern const int NOT_AN_AGGREGATE;
extern const int NOT_IMPLEMENTED;
extern const int UNEXPECTED_EXPRESSION;
extern const int UNSUPPORTED_METHOD;
extern const int TOO_DEEP_SUBQUERIES;
}
namespace
{
void validateFilter(const QueryTreeNodePtr & filter_node, std::string_view exception_place_message, const QueryTreeNodePtr & query_node)
{
DataTypePtr filter_node_result_type;
try
{
filter_node_result_type = filter_node->getResultType();
}
catch (const DB::Exception &e)
{
if (e.code() != ErrorCodes::UNSUPPORTED_METHOD)
e.rethrow();
}
if (!filter_node_result_type)
throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION,
"Unexpected expression '{}' in filter in {}. In query {}",
filter_node->formatASTForErrorMessage(),
exception_place_message,
query_node->formatASTForErrorMessage());
if (!filter_node_result_type->canBeUsedInBooleanContext())
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER,
"Invalid type for filter in {}: {}. In query {}",
exception_place_message,
filter_node_result_type->getName(),
query_node->formatASTForErrorMessage());
}
}
void validateFilters(const QueryTreeNodePtr & query_node)
{
const auto & query_node_typed = query_node->as<QueryNode &>();
if (query_node_typed.hasPrewhere())
{
validateFilter(query_node_typed.getPrewhere(), "PREWHERE", query_node);
assertNoFunctionNodes(query_node_typed.getPrewhere(),
"arrayJoin",
ErrorCodes::ILLEGAL_PREWHERE,
"ARRAY JOIN",
"in PREWHERE");
}
if (query_node_typed.hasWhere())
validateFilter(query_node_typed.getWhere(), "WHERE", query_node);
if (query_node_typed.hasHaving())
validateFilter(query_node_typed.getHaving(), "HAVING", query_node);
if (query_node_typed.hasQualify())
validateFilter(query_node_typed.getQualify(), "QUALIFY", query_node);
}
static bool areColumnSourcesEqual(const QueryTreeNodePtr & lhs, const QueryTreeNodePtr & rhs)
{
using NodePair = std::pair<const IQueryTreeNode *, const IQueryTreeNode *>;
std::vector<NodePair> nodes_to_process;
nodes_to_process.emplace_back(lhs.get(), rhs.get());
while (!nodes_to_process.empty())
{
const auto [lhs_node, rhs_node] = nodes_to_process.back();
nodes_to_process.pop_back();
if (lhs_node->getNodeType() != rhs_node->getNodeType())
return false;
if (lhs_node->getNodeType() == QueryTreeNodeType::COLUMN)
{
const auto * lhs_column_node = lhs_node->as<ColumnNode>();
const auto * rhs_column_node = rhs_node->as<ColumnNode>();
if (!lhs_column_node->getColumnSource()->isEqual(*rhs_column_node->getColumnSource()))
return false;
}
const auto & lhs_children = lhs_node->getChildren();
const auto & rhs_children = rhs_node->getChildren();
if (lhs_children.size() != rhs_children.size())
return false;
for (size_t i = 0; i < lhs_children.size(); ++i)
{
const auto & lhs_child = lhs_children[i];
const auto & rhs_child = rhs_children[i];
if (!lhs_child && !rhs_child)
continue;
if (lhs_child && !rhs_child)
return false;
if (!lhs_child && rhs_child)
return false;
nodes_to_process.emplace_back(lhs_child.get(), rhs_child.get());
}
}
return true;
}
bool compareGroupByKeys(const QueryTreeNodePtr & node, const QueryTreeNodePtr & group_by_key_node)
{
if (node->isEqual(*group_by_key_node, {.compare_aliases = false}))
{
/** Column sources should be compared with aliases for correct GROUP BY keys validation,
* otherwise t2.x and t1.x will be considered as the same column:
* SELECT t2.x FROM t1 JOIN t1 as t2 ON t1.x = t2.x GROUP BY t1.x;
*/
if (areColumnSourcesEqual(node, group_by_key_node))
return true;
}
return false;
}
namespace
{
class ValidateGroupByColumnsVisitor : public ConstInDepthQueryTreeVisitor<ValidateGroupByColumnsVisitor>
{
public:
explicit ValidateGroupByColumnsVisitor(
const QueryTreeNodes & group_by_keys_nodes_,
const QueryTreeNodes & original_group_by_keys_nodes_,
const QueryTreeNodePtr & query_node_)
: group_by_keys_nodes(group_by_keys_nodes_)
, original_group_by_keys_nodes(original_group_by_keys_nodes_)
, query_node(query_node_)
{}
void visitImpl(const QueryTreeNodePtr & node)
{
auto query_tree_node_type = node->getNodeType();
if (query_tree_node_type == QueryTreeNodeType::CONSTANT ||
query_tree_node_type == QueryTreeNodeType::SORT ||
query_tree_node_type == QueryTreeNodeType::INTERPOLATE)
return;
if (nodeIsAggregateFunctionOrInGroupByKeys(node))
return;
auto * function_node = node->as<FunctionNode>();
if (function_node && function_node->getFunctionName() == "grouping")
{
auto & grouping_function_arguments_nodes = function_node->getArguments().getNodes();
for (auto & grouping_function_arguments_node : grouping_function_arguments_nodes)
{
bool found_argument_in_group_by_keys = false;
/// Arguments of the `grouping` function only identify GROUP BY keys, so they are
/// not converted to Nullable when `group_by_use_nulls` is enabled and must be
/// compared with the keys in their original form.
for (const auto & group_by_key_node : original_group_by_keys_nodes)
{
if (grouping_function_arguments_node->isEqual(*group_by_key_node))
{
found_argument_in_group_by_keys = true;
break;
}
}
if (!found_argument_in_group_by_keys)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"GROUPING function argument {} is not in GROUP BY keys. In query {}",
grouping_function_arguments_node->formatASTForErrorMessage(),
query_node->formatASTForErrorMessage());
}
return;
}
auto * column_node = node->as<ColumnNode>();
if (!column_node)
return;
auto column_node_source = column_node->getColumnSource();
if (column_node_source->getNodeType() == QueryTreeNodeType::LAMBDA_ARGS)
return;
if (column_node_source->getNodeType() == QueryTreeNodeType::INTERPOLATE)
return;
throw Exception(ErrorCodes::NOT_AN_AGGREGATE,
"Column '{}' is not under aggregate function and not in GROUP BY keys. In query {}",
column_node->formatConvertedASTForErrorMessage(),
query_node->formatASTForErrorMessage());
}
bool needChildVisit(const QueryTreeNodePtr & parent_node, const QueryTreeNodePtr & child_node)
{
/// Arguments of the `grouping` function are validated in visitImpl against the keys
/// in the original form. They must not be visited as ordinary expressions: when
/// `group_by_use_nulls` is enabled, they are not converted to Nullable and would not
/// match the converted GROUP BY keys.
if (auto * parent_function_node = parent_node->as<FunctionNode>())
if (parent_function_node->getFunctionName() == "grouping")
return false;
if (nodeIsAggregateFunctionOrInGroupByKeys(parent_node))
return false;
auto child_node_type = child_node->getNodeType();
return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION);
}
private:
bool nodeIsAggregateFunctionOrInGroupByKeys(const QueryTreeNodePtr & node) const
{
if (auto * function_node = node->as<FunctionNode>())
if (function_node->isAggregateFunction())
return true;
for (const auto & group_by_key_node : group_by_keys_nodes)
{
if (compareGroupByKeys(node, group_by_key_node))
return true;
}
return false;
}
const QueryTreeNodes & group_by_keys_nodes;
const QueryTreeNodes & original_group_by_keys_nodes;
const QueryTreeNodePtr & query_node;
};
}
void validateAggregates(const QueryTreeNodePtr & query_node, AggregatesValidationParams params)
{
const auto & query_node_typed = query_node->as<QueryNode &>();
auto join_tree_node_type = query_node_typed.getJoinTreeNode()->getNodeType();
bool join_tree_is_subquery = join_tree_node_type == QueryTreeNodeType::QUERY || join_tree_node_type == QueryTreeNodeType::UNION;
if (!join_tree_is_subquery)
{
assertNoAggregateFunctionNodes(query_node_typed.getJoinTreeNode(), "in JOIN TREE");
assertNoGroupingFunctionNodes(query_node_typed.getJoinTreeNode(), "in JOIN TREE");
assertNoWindowFunctionNodes(query_node_typed.getJoinTreeNode(), "in JOIN TREE");
}
/// `SELECT count() AS c FROM t WHERE c > 1` is the common shape: the alias is expanded before this
/// check, so the user is told about an aggregate in WHERE that they never wrote. Name the clause that
/// does accept it. The text is shared with the old analyzer's copy of the check.
static const String use_having_hint = AGGREGATE_IN_WHERE_HINT;
if (query_node_typed.hasWhere())
{
assertNoAggregateFunctionNodes(query_node_typed.getWhere(), "in WHERE", use_having_hint);
assertNoGroupingFunctionNodes(query_node_typed.getWhere(), "in WHERE");
assertNoWindowFunctionNodes(query_node_typed.getWhere(), "in WHERE");
}
if (query_node_typed.hasPrewhere())
{
assertNoAggregateFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE", use_having_hint);
assertNoGroupingFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE");
assertNoWindowFunctionNodes(query_node_typed.getPrewhere(), "in PREWHERE");
}
if (query_node_typed.hasHaving())
assertNoWindowFunctionNodes(query_node_typed.getHaving(), "in HAVING");
if (query_node_typed.hasWindow())
assertNoWindowFunctionNodes(query_node_typed.getWindowNode(), "in WINDOW");
QueryTreeNodes aggregate_function_nodes;
QueryTreeNodes window_function_nodes;
collectAggregateFunctionNodes(query_node, aggregate_function_nodes);
collectWindowFunctionNodes(query_node, window_function_nodes);
if (query_node_typed.hasGroupBy())
{
assertNoAggregateFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY");
assertNoGroupingFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY");
assertNoWindowFunctionNodes(query_node_typed.getGroupByNode(), "in GROUP BY");
}
for (auto & aggregate_function_node : aggregate_function_nodes)
{
auto & aggregate_function_node_typed = aggregate_function_node->as<FunctionNode &>();
assertNoAggregateFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function");
assertNoGroupingFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside another aggregate function");
assertNoWindowFunctionNodes(aggregate_function_node_typed.getArgumentsNode(), "inside an aggregate function");
}
for (auto & window_function_node : window_function_nodes)
{
auto & window_function_node_typed = window_function_node->as<FunctionNode &>();
assertNoWindowFunctionNodes(window_function_node_typed.getArgumentsNode(), "inside another window function");
if (query_node_typed.hasWindow())
assertNoWindowFunctionNodes(window_function_node_typed.getWindowNode(), "inside window definition");
}
/// GROUP BY keys in the form in which expressions equal to them appear in the query tree
/// (converted to Nullable when `group_by_use_nulls` is enabled), and in the original form
/// (for `grouping` function arguments, which are never converted).
QueryTreeNodes group_by_keys_nodes;
QueryTreeNodes original_group_by_keys_nodes;
group_by_keys_nodes.reserve(query_node_typed.getGroupBy().getNodes().size());
original_group_by_keys_nodes.reserve(query_node_typed.getGroupBy().getNodes().size());
for (const auto & node : query_node_typed.getGroupBy().getNodes())
{
if (query_node_typed.isGroupByWithGroupingSets())
{
auto & grouping_set_keys = node->as<ListNode &>();
for (auto & grouping_set_key : grouping_set_keys.getNodes())
{
original_group_by_keys_nodes.push_back(grouping_set_key);
group_by_keys_nodes.push_back(grouping_set_key->clone());
if (params.group_by_use_nulls)
group_by_keys_nodes.back()->convertToNullable();
}
}
else
{
original_group_by_keys_nodes.push_back(node);
group_by_keys_nodes.push_back(node->clone());
if (params.group_by_use_nulls)
group_by_keys_nodes.back()->convertToNullable();
}
}
if (query_node_typed.getGroupBy().getNodes().empty())
{
if (query_node_typed.hasHaving())
assertNoGroupingFunctionNodes(query_node_typed.getHaving(), "in HAVING without GROUP BY");
if (query_node_typed.hasOrderBy())
assertNoGroupingFunctionNodes(query_node_typed.getOrderByNode(), "in ORDER BY without GROUP BY");
assertNoGroupingFunctionNodes(query_node_typed.getProjectionNode(), "in SELECT without GROUP BY");
}
bool has_aggregation = !query_node_typed.getGroupBy().getNodes().empty() || !aggregate_function_nodes.empty();
if (has_aggregation)
{
ValidateGroupByColumnsVisitor validate_group_by_columns_visitor(group_by_keys_nodes, original_group_by_keys_nodes, query_node);
if (query_node_typed.hasHaving())
validate_group_by_columns_visitor.visit(query_node_typed.getHaving());
if (query_node_typed.hasQualify())
validate_group_by_columns_visitor.visit(query_node_typed.getQualify());
if (query_node_typed.hasOrderBy())
validate_group_by_columns_visitor.visit(query_node_typed.getOrderByNode());
if (query_node_typed.hasInterpolate())
validate_group_by_columns_visitor.visit(query_node_typed.getInterpolate());
if (query_node_typed.hasLimitBy())
validate_group_by_columns_visitor.visit(query_node_typed.getLimitByNode());
if (query_node_typed.hasLimitAfter())
validate_group_by_columns_visitor.visit(query_node_typed.getLimitAfter());
if (query_node_typed.hasLimitUntil())
validate_group_by_columns_visitor.visit(query_node_typed.getLimitUntil());
validate_group_by_columns_visitor.visit(query_node_typed.getProjectionNode());
}
bool aggregation_with_rollup_or_cube_or_grouping_sets = query_node_typed.isGroupByWithRollup() ||
query_node_typed.isGroupByWithCube() ||
query_node_typed.isGroupByWithGroupingSets();
if (!has_aggregation && (query_node_typed.isGroupByWithTotals() || aggregation_with_rollup_or_cube_or_grouping_sets))
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS, ROLLUP, CUBE or GROUPING SETS are not supported without aggregation");
}
namespace
{
class ValidateFunctionNodesVisitor : public ConstInDepthQueryTreeVisitor<ValidateFunctionNodesVisitor>
{
public:
explicit ValidateFunctionNodesVisitor(std::string_view function_name_,
int exception_code_,
std::string_view exception_function_name_,
std::string_view exception_place_message_)
: function_name(function_name_)
, exception_code(exception_code_)
, exception_function_name(exception_function_name_)
, exception_place_message(exception_place_message_)
{}
void visitImpl(const QueryTreeNodePtr & node)
{
auto * function_node = node->as<FunctionNode>();
if (function_node && function_node->getFunctionName() == function_name)
throw Exception(exception_code,
"{} function {} is found {} in query",
exception_function_name,
function_node->formatASTForErrorMessage(),
exception_place_message);
}
static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node)
{
auto child_node_type = child_node->getNodeType();
return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION);
}
private:
std::string_view function_name;
int exception_code = 0;
std::string_view exception_function_name;
std::string_view exception_place_message;
};
}
void assertNoFunctionNodes(const QueryTreeNodePtr & node,
std::string_view function_name,
int exception_code,
std::string_view exception_function_name,
std::string_view exception_place_message)
{
ValidateFunctionNodesVisitor visitor(function_name, exception_code, exception_function_name, exception_place_message);
visitor.visit(node);
}
void validateTreeSize(const QueryTreeNodePtr & node,
size_t max_size,
std::unordered_map<QueryTreeNodePtr, size_t> & node_to_tree_size)
{
size_t tree_size = 0;
std::vector<std::pair<QueryTreeNodePtr, bool>> nodes_to_process;
nodes_to_process.emplace_back(node, false);
while (!nodes_to_process.empty())
{
const auto [node_to_process, processed_children] = nodes_to_process.back();
nodes_to_process.pop_back();
if (processed_children)
{
++tree_size;
size_t subtree_size = 1;
for (const auto & node_to_process_child : node_to_process->getChildren())
{
if (!node_to_process_child)
continue;
subtree_size += node_to_tree_size[node_to_process_child];
}
auto * constant_node = node_to_process->as<ConstantNode>();
if (constant_node && constant_node->hasSourceExpression())
subtree_size += node_to_tree_size[constant_node->getSourceExpression()];
node_to_tree_size.emplace(node_to_process, subtree_size);
continue;
}
auto node_to_size_it = node_to_tree_size.find(node_to_process);
if (node_to_size_it != node_to_tree_size.end())
{
tree_size += node_to_size_it->second;
continue;
}
nodes_to_process.emplace_back(node_to_process, true);
for (const auto & node_to_process_child : node_to_process->getChildren())
{
if (!node_to_process_child)
continue;
nodes_to_process.emplace_back(node_to_process_child, false);
}
auto * constant_node = node_to_process->as<ConstantNode>();
if (constant_node && constant_node->hasSourceExpression())
nodes_to_process.emplace_back(constant_node->getSourceExpression(), false);
}
if (tree_size > max_size)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Query tree is too big. Maximum: {}",
max_size);
}
void validateSubqueryDepth(const QueryTreeNodePtr &node, size_t initial_subquery_depth, size_t max_subquery_depth)
{
if (!max_subquery_depth)
return;
size_t current_depth = initial_subquery_depth;
std::vector<std::pair<QueryTreeNodePtr, bool>> nodes_to_process;
nodes_to_process.emplace_back(node, false);
traverseQueryTree(node, Everything{},
[¤t_depth, max_subquery_depth](auto & current_node)
{
if (current_node->getNodeType() == QueryTreeNodeType::QUERY)
++current_depth;
if (current_depth > max_subquery_depth)
throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, "Too deep subqueries. Maximum: {}", max_subquery_depth);
},
[¤t_depth](const QueryTreeNodePtr & current_node)
{
if (current_node->getNodeType() == QueryTreeNodeType::QUERY)
--current_depth;
});
}
/// A table in a database with `lazy_load_tables`, and a permanent table created `AS` a table function
/// (`CREATE TABLE t AS view(SELECT ...)`), is attached as a `StorageProxy` around the real storage.
/// A proxy forwards `isRemote` and `readsFromOtherTables` but not its type, and the look-throughs in
/// `readsFromRemoteTable` are keyed on the concrete storage - so a lazily loaded `Merge` would fall
/// through to the dependency walk, which records nothing for `Merge`. `StorageTableFunctionProxy` is
/// worse still: it reports `isView() == false` outright, and the `StorageView` it wraps answers the
/// `IStorage` default `readsFromOtherTables() == false`, so neither predicate recognizes such a table
/// as opaque unless the proxy is resolved first.
///
/// Resolving the nested storage costs nothing at every call site below: each one is reached only after
/// an `isRemote` call, which already materialized it.
static StoragePtr unwrapStorageProxy(const StoragePtr & storage)
{
static constexpr size_t max_proxy_depth = 16;
StoragePtr nested_storage = storage;
for (size_t i = 0; i < max_proxy_depth && nested_storage; ++i)
{
const auto * proxy = dynamic_cast<const StorageProxy *>(nested_storage.get());
if (!proxy)
break;
nested_storage = proxy->getNested();
}
return nested_storage;
}
/// Whether a table's own storage is local but reading it reads other tables, so what it reaches has to
/// be resolved through the catalog's dependency graph.
static bool isOpaqueTable(const StoragePtr & storage)
{
auto nested_storage = unwrapStorageProxy(storage);
return nested_storage && (nested_storage->isView() || nested_storage->readsFromOtherTables());
}
/// Whether reading this table reaches a remote table. A `VIEW` is an opaque `TABLE` node at this
/// point - the analyzer expands it later, inside `StorageView::read` - so its own `isRemote` says
/// nothing about what it reads. Follow the referential dependencies the catalog records for such a
/// table instead. Without this, a correlated subquery over a view of a `Distributed` table passed the
/// check below and then broke a planner invariant (`Column identifier ... is already registered`,
/// a `LOGICAL_ERROR` that aborts a debug build) instead of being refused.
///
/// Not every remote read is reachable this way: `DDLDependencyVisitor` records no dependency for the
/// `remote` and `cluster` table functions, so a view over those stays invisible here, exactly as it
/// is today. Those spellings execute rather than being refused, so nothing regresses.
static bool readsFromRemoteTable(
const StoragePtr & storage,
const StorageID & table_id,
const ContextPtr & context,
std::unordered_set<String> & visited,
size_t depth)
{
static constexpr size_t max_dependency_depth = 16;
if (!storage)
return false;
if (storage->isRemote())
return true;
if (!context || depth >= max_dependency_depth)
return false;
/// A cycle in the graph would otherwise be bounded only by the depth cap, and every `Merge` level
/// iterates over databases. A storage produced by a table function may not be in the catalog and then
/// has nothing to key on; the depth cap still bounds those.
if (!table_id.table_name.empty() && !visited.insert(table_id.getFullTableName()).second)
return false;
StoragePtr nested_storage = unwrapStorageProxy(storage);
/// Reading a materialized view only ever reads its target table, so follow that target rather than the
/// referential dependencies: those also include the `SELECT` source, which is read at insert time only,
/// and following it would refuse a materialized view with a `Distributed` source and a local target - a
/// query that works. `StorageMaterializedView::isRemote`, checked above, only asks the target about
/// itself, which misses a target that is itself a view over a `Distributed` table.
if (const auto * materialized_view = nested_storage->as<StorageMaterializedView>())
return readsFromRemoteTable(
materialized_view->tryGetTargetTable(), materialized_view->getTargetTableId(), context, visited, depth + 1);
/// `Merge` matches its source tables by a pattern resolved at read time, so the catalog records no
/// referential dependency for it. `StorageMerge::isRemote`, checked above, asks every matched source
/// only about itself, which misses a source that is a view over a `Distributed` table. Walk the
/// currently matched sources instead.
if (const auto * storage_merge = nested_storage->as<StorageMerge>())
{
return storage_merge->hasChildTable([&](const StoragePtr & source)
{
return readsFromRemoteTable(source, source->getStorageID(), context, visited, depth + 1);
});
}
if (!(nested_storage->isView() || nested_storage->readsFromOtherTables()))
return false;
/// A table function such as `view(SELECT ...)` also wraps a `StorageView`, but it is not in the catalog,
/// so there are no dependencies to follow.
if (table_id.table_name.empty())
return false;
for (const auto & dependency : DatabaseCatalog::instance().getReferentialDependencies(table_id))
{
auto dependency_storage = DatabaseCatalog::instance().tryGetTable(dependency, context);
if (readsFromRemoteTable(dependency_storage, dependency, context, visited, depth + 1))
return true;
}
return false;
}
void validateCorrelatedSubqueries(const QueryTreeNodePtr & node, const ContextPtr & context)
{
bool has_remote = false;
bool has_correlated_subquery = false;
QueryTreeNodes nodes_to_process = { node };
/// Tables whose own storage is local but that read from other tables. Resolving what they reach
/// takes the catalog's dependency graph, so it is deferred to the end and only done when the query
/// actually has a correlated subquery.
std::vector<std::pair<StoragePtr, StorageID>> opaque_tables;
while (!nodes_to_process.empty())
{
auto current_node = nodes_to_process.back();
nodes_to_process.pop_back();
switch (current_node->getNodeType())
{
case QueryTreeNodeType::QUERY:
{
auto & query_node = current_node->as<QueryNode &>();
if (query_node.isCorrelated())
has_correlated_subquery = true;
break;
}
case QueryTreeNodeType::UNION:
{
auto & union_node = current_node->as<UnionNode &>();
if (union_node.isCorrelated())
has_correlated_subquery = true;
break;
}
case QueryTreeNodeType::TABLE:
{
auto & table_node = current_node->as<TableNode &>();
const auto & storage = table_node.getStorage();
if (storage && storage->isRemote())
has_remote = true;
else if (storage && isOpaqueTable(storage))
opaque_tables.emplace_back(storage, table_node.getStorageID());
break;
}
case QueryTreeNodeType::TABLE_FUNCTION:
{
auto & table_function_node = current_node->as<TableFunctionNode &>();
const auto & storage = table_function_node.getStorage();
if (storage && storage->isRemote())
has_remote = true;
/// A parameterized view is resolved as a `TableFunctionNode` wrapping the real `StorageView`,
/// not as a `TableNode`, so it needs the same look-through as an ordinary view. The `merge`
/// table function arrives the same way.
else if (storage && isOpaqueTable(storage))
opaque_tables.emplace_back(storage, table_function_node.getStorageID());
break;
}
default:
break;
}
if (has_remote && has_correlated_subquery)
throw Exception(ErrorCodes::NOT_IMPLEMENTED,
"Correlated subqueries are not supported with remote tables. In query {}",
node->formatASTForErrorMessage());
for (const auto & child : current_node->getChildren())
{
if (child)
nodes_to_process.push_back(child);
}
}
if (!has_correlated_subquery)
return;
for (const auto & [storage, storage_id] : opaque_tables)
{
std::unordered_set<String> visited;
if (readsFromRemoteTable(storage, storage_id, context, visited, 0))
throw Exception(ErrorCodes::NOT_IMPLEMENTED,
"Correlated subqueries are not supported with remote tables. In query {}",
node->formatASTForErrorMessage());
}
}
void validateFromClause(const QueryTreeNodePtr & node)
{
const auto & root_query_node = node->as<QueryNode &>();
auto correlated_columns_set = root_query_node.getCorrelatedColumnsSet();
std::vector<QueryTreeNodePtr> nodes_to_process = { root_query_node.getJoinTreeNode() };
while (!nodes_to_process.empty())
{
auto node_to_process = std::move(nodes_to_process.back());
nodes_to_process.pop_back();
auto node_type = node_to_process->getNodeType();
switch (node_type)
{
case QueryTreeNodeType::TABLE:
[[fallthrough]];
case QueryTreeNodeType::TABLE_FUNCTION:
break;
case QueryTreeNodeType::QUERY:
{
auto & query_node = node_to_process->as<QueryNode &>();
const auto & correlated_columns = query_node.getCorrelatedColumns();
for (const auto & column : correlated_columns)
{
if (!correlated_columns_set.contains(std::static_pointer_cast<ColumnNode>(column)))
throw Exception(ErrorCodes::NOT_IMPLEMENTED,
"Lateral joins are not supported. Correlated column '{}' is found in the FROM clause. In query {}",
column->formatASTForErrorMessage(),
node->formatASTForErrorMessage());
}
break;
}
case QueryTreeNodeType::UNION:
{
for (const auto & union_node : node_to_process->as<UnionNode>()->getQueries().getNodes())
nodes_to_process.push_back(union_node);
break;
}
case QueryTreeNodeType::ARRAY_JOIN:
{
auto & array_join_node = node_to_process->as<ArrayJoinNode &>();
nodes_to_process.push_back(array_join_node.getTableExpressionNode());
break;
}
case QueryTreeNodeType::CROSS_JOIN:
{
auto & join_node = node_to_process->as<CrossJoinNode &>();
for (const auto & expr : std::ranges::reverse_view(join_node.getTableExpressions()))
nodes_to_process.push_back(expr);
break;
}
case QueryTreeNodeType::JOIN:
{
auto & join_node = node_to_process->as<JoinNode &>();
nodes_to_process.push_back(join_node.getRightTableExpressionNode());
nodes_to_process.push_back(join_node.getLeftTableExpressionNode());
break;
}
default:
{
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Unexpected node type for table expression. "
"Expected table, table function, query, union, join or array join. Actual {}",
node_to_process->getNodeTypeName());
}
}
}
}
}