Skip to content

fix(deps): vendor sqlglot bigquery dialect and remove package dependency #2354

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chelsea-lin wants to merge 3 commits into
base: main
Choose a base branch
from
+47,606 −144
Open

fix(deps): vendor sqlglot bigquery dialect and remove package dependency #2354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6d46ce3
deps: vendor sqlglot codes and remove dependency
chelsea-lin Dec 23, 2025
fe49168
fix mypy lint format
chelsea-lin Dec 23, 2025
d6ab6bc
exclude sqlglot from doctest
chelsea-lin Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .github/ISSUE_TEMPLATE/bug_report.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,12 @@ import bigframes
import google.cloud.bigquery
import pandas
import pyarrow
import sqlglot

print(f"Python: {sys.version}")
print(f"bigframes=={bigframes.__version__}")
print(f"google-cloud-bigquery=={google.cloud.bigquery.__version__}")
print(f"pandas=={pandas.__version__}")
print(f"pyarrow=={pyarrow.__version__}")
print(f"sqlglot=={sqlglot.__version__}")
```

#### Steps to reproduce
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregate_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.
from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes.core import agg_expressions, window_spec
from bigframes.core.compile.sqlglot.aggregations import (
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregations/binary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import typing

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes.core import window_spec
import bigframes.core.compile.sqlglot.aggregations.op_registration as reg
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregations/nullary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import typing

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes.core import window_spec
import bigframes.core.compile.sqlglot.aggregations.op_registration as reg
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregations/op_registration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import typing

from sqlglot import expressions as sge
from bigframes_vendored.sqlglot import expressions as sge

from bigframes.operations import aggregations as agg_ops

Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregations/ordered_unary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

import bigframes.core.compile.sqlglot.aggregations.op_registration as reg
import bigframes.core.compile.sqlglot.expressions.typed_expr as typed_expr
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregations/unary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

import typing

import bigframes_vendored.sqlglot.expressions as sge
import pandas as pd
import sqlglot.expressions as sge

from bigframes import dtypes
from bigframes.core import window_spec
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/aggregations/windows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

import typing

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes.core import utils, window_spec
import bigframes.core.compile.sqlglot.scalar_compiler as scalar_compiler
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import functools
import typing

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes.core import (
agg_expressions,
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/ai_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

from dataclasses import asdict

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot import scalar_compiler
Expand Down
4 changes: 2 additions & 2 deletions bigframes/core/compile/sqlglot/expressions/array_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

import typing

import sqlglot as sg
import sqlglot.expressions as sge
import bigframes_vendored.sqlglot as sg
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot.expressions.string_ops import (
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/blob_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/bool_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/comparison_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

import typing

import bigframes_vendored.sqlglot.expressions as sge
import pandas as pd
import sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import math

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

_ZERO = sge.Cast(this=sge.convert(0), to="INT64")
_NAN = sge.Cast(this=sge.convert("NaN"), to="FLOAT64")
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/date_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/datetime_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
4 changes: 2 additions & 2 deletions bigframes/core/compile/sqlglot/expressions/generic_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@

from __future__ import annotations

import sqlglot as sg
import sqlglot.expressions as sge
import bigframes_vendored.sqlglot as sg
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/geo_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/json_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/numeric_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from __future__ import annotations

import bigframes_vendored.constants as bf_constants
import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/string_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import functools
import typing

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/struct_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@

import typing

import bigframes_vendored.sqlglot.expressions as sge
import pandas as pd
import pyarrow as pa
import sqlglot.expressions as sge

from bigframes import operations as ops
from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/timedelta_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from __future__ import annotations

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes
from bigframes import operations as ops
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/expressions/typed_expr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import dataclasses

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes import dtypes

Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/scalar_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import functools
import typing

import sqlglot.expressions as sge
import bigframes_vendored.sqlglot.expressions as sge

from bigframes.core.compile.sqlglot.expressions.typed_expr import TypedExpr
import bigframes.core.compile.sqlglot.sqlglot_ir as ir
Expand Down
7 changes: 3 additions & 4 deletions bigframes/core/compile/sqlglot/sqlglot_ir.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,12 @@
import functools
import typing

import bigframes_vendored.sqlglot as sg
import bigframes_vendored.sqlglot.expressions as sge
from google.cloud import bigquery
import numpy as np
import pandas as pd
import pyarrow as pa
import sqlglot as sg
import sqlglot.dialects.bigquery
import sqlglot.expressions as sge

from bigframes import dtypes
from bigframes.core import guid, local_data, schema, utils
Expand All @@ -48,7 +47,7 @@ class SQLGlotIR:
expr: sge.Select = sg.select()
"""The SQLGlot expression representing the query."""

dialect = sqlglot.dialects.bigquery.BigQuery
dialect = sg.dialects.bigquery.BigQuery
"""The SQL dialect used for generation."""

quoted: bool = True
Expand Down
2 changes: 1 addition & 1 deletion bigframes/core/compile/sqlglot/sqlglot_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
import typing

import bigframes_vendored.constants as constants
import bigframes_vendored.sqlglot as sg
import numpy as np
import pandas as pd
import pyarrow as pa
import sqlglot as sg

import bigframes.dtypes

Expand Down
2 changes: 2 additions & 0 deletions noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,8 @@ def doctest(session: nox.sessions.Session):
"--ignore",
"third_party/bigframes_vendored/ibis",
"--ignore",
"third_party/bigframes_vendored/sqlglot",
"--ignore",
"bigframes/core/compile/polars",
"--ignore",
"bigframes/testing",
Expand Down
2 changes: 0 additions & 2 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@
"pydata-google-auth >=1.8.2",
"requests >=2.27.1",
"shapely >=1.8.5",
# 25.20.0 introduces this fix https://github.com/TobikoData/sqlmesh/issues/3095 for rtrim/ltrim.
"sqlglot >=25.20.0",
"tabulate >=0.9",
"ipywidgets >=7.7.1",
"humanize >=4.6.0",
Expand Down
1 change: 0 additions & 1 deletion testing/constraints-3.11.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,7 +520,6 @@ sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==2.0.0
sphinxcontrib-serializinghtml==2.0.0
SQLAlchemy==2.0.42
sqlglot==25.20.2
sqlparse==0.5.3
srsly==2.5.1
stanio==0.5.1
Expand Down
1 change: 0 additions & 1 deletion testing/constraints-3.9.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ pydata-google-auth==1.8.2
requests==2.27.1
scikit-learn==1.2.2
shapely==1.8.5
sqlglot==25.20.0
tabulate==0.9
ipywidgets==7.7.1
humanize==4.6.0
Expand Down
7 changes: 0 additions & 7 deletions tests/system/small/bigquery/test_ai.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@

from unittest import mock

from packaging import version
import pandas as pd
import pyarrow as pa
import pytest
import sqlglot

from bigframes import dataframe, dtypes, series
import bigframes.bigquery as bbq
Expand Down Expand Up @@ -67,11 +65,6 @@ def test_ai_function_string_input(session):


def test_ai_function_compile_model_params(session):
if version.Version(sqlglot.__version__) < version.Version("25.18.0"):
pytest.skip(
"Skip test because SQLGLot cannot compile model params to JSON at this version."
)

s1 = bpd.Series(["apple", "bear"], session=session)
s2 = bpd.Series(["fruit", "tree"], session=session)
prompt = (s1, " is a ", s2)
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/core/compile/sqlglot/aggregations/test_op_registration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from bigframes_vendored.sqlglot import expressions as sge
import pytest
from sqlglot import expressions as sge

from bigframes.core.compile.sqlglot.aggregations import op_registration
from bigframes.operations import aggregations as agg_ops
Expand Down
13 changes: 0 additions & 13 deletions tests/unit/core/compile/sqlglot/aggregations/test_ordered_unary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import typing

import pytest
Expand Down Expand Up @@ -47,12 +46,6 @@ def _apply_ordered_unary_agg_ops(


def test_array_agg(scalar_types_df: bpd.DataFrame, snapshot):
# TODO: Verify "NULL LAST" syntax issue on Python < 3.12
if sys.version_info < (3, 12):
pytest.skip(
"Skipping test due to inconsistent SQL formatting on Python < 3.12.",
)

col_name = "int64_col"
bf_df = scalar_types_df[[col_name]]
agg_expr = agg_ops.ArrayAggOp().as_expr(col_name)
Expand All @@ -64,12 +57,6 @@ def test_array_agg(scalar_types_df: bpd.DataFrame, snapshot):


def test_string_agg(scalar_types_df: bpd.DataFrame, snapshot):
# TODO: Verify "NULL LAST" syntax issue on Python < 3.12
if sys.version_info < (3, 12):
pytest.skip(
"Skipping test due to inconsistent SQL formatting on Python < 3.12.",
)

col_name = "string_col"
bf_df = scalar_types_df[[col_name]]
agg_expr = agg_ops.StringAggOp(sep=",").as_expr(col_name)
Expand Down
Loading