googleapis · chalmerlowe · Mar 22, 2023 · Oct 25, 2022 · Jan 23, 2023 · Feb 10, 2023
@@ -81,6 +81,20 @@ Windows
     <your-env>\Scripts\activate
     <your-env>\Scripts\pip.exe install sqlalchemy-bigquery
 
+
+Installations when processing large datasets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When handling large datasets, you may see speed increases by also installing the
+`bqstorage` dependencies. See the instructions above about creating a virtual 
+environment and then install `sqlalchemy-bigquery` using the `bqstorage` extras:
+
+.. code-block:: console
+
+    source <your-env>/bin/activate
+    <your-env>/bin/pip install sqlalchemy-bigquery[bqstorage]
+
+
 Usage
 -----
 

@@ -50,10 +50,12 @@
     "3.8": [
         "tests",
         "alembic",
+        "bqstorage",
     ],
     "3.11": [
         "tests",
         "geography",
+        "bqstorage",
     ],
 }
 
@@ -73,10 +75,12 @@
     "3.8": [
         "tests",
         "alembic",
+        "bqstorage",
     ],
     "3.11": [
         "tests",
         "geography",
+        "bqstorage",
     ],
 }
 
@@ -179,14 +183,22 @@ def install_unittest_dependencies(session, *constraints):
         session.install("-e", ".", *constraints)
 
 
-def default(session):
+def default(session, install_extras=True):
     # Install all test dependencies, then install this package in-place.
 
     constraints_path = str(
         CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
     )
     install_unittest_dependencies(session, "-c", constraints_path)
 
+    if install_extras and session.python == "3.11":
+        install_target = ".[geography,alembic,tests,bqstorage]"
+    elif install_extras:
+        install_target = ".[all]"
+    else:
+        install_target = "."
+    session.install("-e", install_target, "-c", constraints_path)
+
     # Run py.test against the unit tests.
     session.run(
         "py.test",
@@ -283,6 +295,51 @@ def system(session):
         )
 
 
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+def system_noextras(session):
+    """Run the system test suite."""
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+    system_test_path = os.path.join("tests", "system.py")
+    system_test_folder_path = os.path.join("tests", "system")
+
+    # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
+    if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
+        session.skip("RUN_SYSTEM_TESTS is set to false, skipping")
+    # Install pyopenssl for mTLS testing.
+    if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true":
+        session.install("pyopenssl")
+
+    system_test_exists = os.path.exists(system_test_path)
+    system_test_folder_exists = os.path.exists(system_test_folder_path)
+    # Sanity check: only run tests if found.
+    if not system_test_exists and not system_test_folder_exists:
+        session.skip("System tests were not found")
+
+    global SYSTEM_TEST_EXTRAS_BY_PYTHON
+    SYSTEM_TEST_EXTRAS_BY_PYTHON = False
+    install_systemtest_dependencies(session, "-c", constraints_path)
+
+    # Run py.test against the system tests.
+    if system_test_exists:
+        session.run(
+            "py.test",
+            "--quiet",
+            f"--junitxml=system_{session.python}_sponge_log.xml",
+            system_test_path,
+            *session.posargs,
+        )
+    if system_test_folder_exists:
+        session.run(
+            "py.test",
+            "--quiet",
+            f"--junitxml=system_{session.python}_sponge_log.xml",
+            system_test_folder_path,
+            *session.posargs,
+        )
+
+
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1])
 def compliance(session):
     """Run the SQLAlchemy dialect-compliance system tests"""

@@ -29,8 +29,8 @@
 # ----------------------------------------------------------------------------
 extras = ["tests"]
 extras_by_python = {
-    "3.8": ["tests", "alembic"],
-    "3.11": ["tests", "geography"],
+    "3.8": ["tests", "alembic", "bqstorage"],
+    "3.11": ["tests", "geography", "bqstorage"],
 }
 templated_files = common.py_library(
     unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"],
@@ -95,6 +95,14 @@
 )
 
 
+s.replace(
+    ["noxfile.py"],
+    r"def default\(session\)",
+    "def default(session, install_extras=True)",    
+)
+
+
+
 
 def place_before(path, text, *before_text, escape=None):
     replacement = "\n".join(before_text) + "\n" + text
@@ -117,6 +125,23 @@ def place_before(path, text, *before_text, escape=None):
 )
 
 
+install_logic = '''
+    if install_extras and session.python == "3.11":
+        install_target = ".[geography,alembic,tests,bqstorage]"
+    elif install_extras:
+        install_target = ".[all]"
+    else:
+        install_target = "."
+    session.install("-e", install_target, "-c", constraints_path)
+'''
+
+place_before(
+    "noxfile.py",
+    "# Run py.test against the unit tests.",
+    install_logic,
+)
+
+
 # Maybe we can get rid of this when we don't need pytest-rerunfailures,
 # which we won't need when BQ retries itself:
 # https://github.com/googleapis/python-bigquery/pull/837
@@ -189,6 +214,63 @@ def compliance(session):
 s.replace(["noxfile.py"], '"alabaster"', '"alabaster", "geoalchemy2", "shapely"')
 
 
+system_noextras = '''
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+def system_noextras(session):
+    """Run the system test suite."""
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+    system_test_path = os.path.join("tests", "system.py")
+    system_test_folder_path = os.path.join("tests", "system")
+
+    # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
+    if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
+        session.skip("RUN_SYSTEM_TESTS is set to false, skipping")
+    # Install pyopenssl for mTLS testing.
+    if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true":
+        session.install("pyopenssl")
+
+    system_test_exists = os.path.exists(system_test_path)
+    system_test_folder_exists = os.path.exists(system_test_folder_path)
+    # Sanity check: only run tests if found.
+    if not system_test_exists and not system_test_folder_exists:
+        session.skip("System tests were not found")
+
+    global SYSTEM_TEST_EXTRAS_BY_PYTHON
+    SYSTEM_TEST_EXTRAS_BY_PYTHON = False
+    install_systemtest_dependencies(session, "-c", constraints_path)
+
+    # Run py.test against the system tests.
+    if system_test_exists:
+        session.run(
+            "py.test",
+            "--quiet",
+            f"--junitxml=system_{session.python}_sponge_log.xml",
+            system_test_path,
+            *session.posargs,
+        )
+    if system_test_folder_exists:
+        session.run(
+            "py.test",
+            "--quiet",
+            f"--junitxml=system_{session.python}_sponge_log.xml",
+            system_test_folder_path,
+            *session.posargs,
+        )
+
+
+'''
+
+
+place_before(
+    "noxfile.py",
+    "@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1])\n"
+    "def compliance(session):", 
+    system_noextras,
+    escape="()[]",
+    )
+
 
 # Add DB config for SQLAlchemy dialect test suite.
 # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/89

@@ -45,11 +45,26 @@ def readme():
         return f.read()
 
 
-extras = dict(
-    geography=["GeoAlchemy2", "shapely"],
-    alembic=["alembic"],
-    tests=["packaging", "pytz"],
-)
+extras = {
+    "geography": ["GeoAlchemy2", "shapely"],
+    "alembic": ["alembic"],
+    "tests": ["packaging", "pytz"],
+    # Keep the no-op bqstorage extra for backward compatibility.
+    # See: https://github.com/googleapis/python-bigquery/issues/757
+    "bqstorage": [
+        "google-cloud-bigquery-storage >= 2.0.0, <3.0.0dev",
+        # Due to an issue in pip's dependency resolver, the `grpc` extra is not
+        # installed, even though `google-cloud-bigquery-storage` specifies it
+        # as `google-api-core[grpc]`. We thus need to explicitly specify it here.
+        # See: https://github.com/googleapis/python-bigquery/issues/83 The
+        # grpc.Channel.close() method isn't added until 1.32.0.
+        # https://github.com/grpc/grpc/pull/15254
+        "grpcio >= 1.47.0, < 2.0dev",
+        "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'",
+        "pyarrow >= 3.0.0",
+    ],
+}
+
 extras["all"] = set(itertools.chain.from_iterable(extras.values()))
 
 setup(
@@ -85,9 +100,7 @@ def readme():
         # https://github.com/googleapis/google-cloud-python/issues/10566
         "google-auth>=1.25.0,<3.0.0dev",  # Work around pip wack.
         "google-cloud-bigquery>=2.25.2,<4.0.0dev",
-        "google-cloud-bigquery-storage>=2.0.0,<3.0.0dev",
         "packaging",
-        "pyarrow>=3.0.0",
         "sqlalchemy>=1.2.0,<2.0.0dev",
         "future",
     ],

@@ -68,7 +68,7 @@
 
 try:
     from .geography import GEOGRAPHY, WKB, WKT  # noqa
-except ImportError:
+except ImportError:  # pragma: NO COVER
     pass
 else:
     __all__.extend(["GEOGRAPHY", "WKB", "WKT"])

@@ -24,7 +24,7 @@
 
 try:
     from .geography import GEOGRAPHY
-except ImportError:
+except ImportError:  # pragma: NO COVER
     pass
 
 from ._struct import STRUCT
@@ -69,7 +69,7 @@
 
 try:
     _type_map["GEOGRAPHY"] = GEOGRAPHY
-except NameError:
+except NameError:  # pragma: NO COVER
     pass
 
 STRUCT_FIELD_TYPES = "RECORD", "STRUCT"

@@ -1055,7 +1055,7 @@ def __init__(self, *args, **kwargs):
 
 try:
     import alembic  # noqa
-except ImportError:
+except ImportError:  # pragma: NO COVER
     pass
 else:
     from alembic.ddl import impl

@@ -6,7 +6,7 @@
 # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
 sqlalchemy==1.2.0
 google-auth==1.25.0
-google-cloud-bigquery==2.25.2
+google-cloud-bigquery==3.3.6
 google-cloud-bigquery-storage==2.0.0
 google-api-core==1.31.5
 pyarrow==3.0.0
@@ -104,7 +104,7 @@ def test_geoalchemy2_core(faux_conn, last_query):
 
     try:
         conn.execute(
-            select([lake_table.c.name, lake_table.c.geog.ST_AREA().label("area")])
+            select([lake_table.c.name, lake_table.c.geog.ST_Area().label("area")])
         )
     except Exception:
         pass  # sqlite had no special functions :)
@@ -169,7 +169,7 @@ def test_calling_st_functions_that_dont_take_geographies(faux_conn, last_query):
     from sqlalchemy import select, func
 
     try:
-        faux_conn.execute(select([func.ST_GEOGFROMTEXT("point(0 0)")]))
+        faux_conn.execute(select([func.ST_GeogFromText("point(0 0)")]))
     except Exception:
         pass  # sqlite had no special functions :)