-
Notifications
You must be signed in to change notification settings - Fork 172
feat(storage): refactor the async code to accomodate JSON and GRPC implementation. #1747
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: async
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| # Copyright 2025 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| """Abstract class for Async JSON and GRPC connection.""" | ||
|
|
||
| import abc | ||
| from google.cloud.storage._http import AGENT_VERSION | ||
| from google.api_core.client_info import ClientInfo | ||
| from google.cloud.storage import __version__ | ||
|
|
||
|
|
||
| class AsyncConnection(abc.ABC): | ||
| """Class for asynchronous connection with JSON and GRPC compatibility. | ||
| This class expose python implementation of interacting with relevant APIs. | ||
| Args: | ||
| client: The client that owns this connection. | ||
| client_info: Information about the client library. | ||
| """ | ||
|
|
||
| def __init__(self, client, client_info=None): | ||
| self._client = client | ||
|
|
||
| if client_info is None: | ||
| client_info = ClientInfo() | ||
|
|
||
| self._client_info = client_info | ||
| if self._client_info.user_agent is None: | ||
| self._client_info.user_agent = AGENT_VERSION | ||
| else: | ||
| self._client_info.user_agent = ( | ||
| f"{self._client_info.user_agent} {AGENT_VERSION}" | ||
| ) | ||
|
Comment on lines
+40
to
+45
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The user agent string construction should be kept consistent with the existing JSON client implementation in if self._client_info.user_agent is None:
self._client_info.user_agent = ""
if AGENT_VERSION not in self._client_info.user_agent:
self._client_info.user_agent += f" {AGENT_VERSION} "References
|
||
| self._client_info.client_library_version = __version__ | ||
| self._extra_headers = {} | ||
|
|
||
| @property | ||
| def extra_headers(self): | ||
| """Returns extra headers to send with every request.""" | ||
| return self._extra_headers | ||
|
|
||
| @extra_headers.setter | ||
| def extra_headers(self, value): | ||
| """Set the extra header property.""" | ||
| self._extra_headers = value | ||
|
|
||
| @property | ||
| def user_agent(self): | ||
| """Returns user_agent for async HTTP transport. | ||
| Returns: | ||
| str: The user agent string. | ||
| """ | ||
| return self._client_info.to_user_agent() | ||
|
|
||
| @user_agent.setter | ||
| def user_agent(self, value): | ||
| """Setter for user_agent in connection.""" | ||
| self._client_info.user_agent = value | ||
|
|
||
| async def close(self): | ||
| pass | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you're deleting
and so on from both Why it's not needed ? |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,25 +14,13 @@ | |
|
|
||
| """Asynchronous client for interacting with Google Cloud Storage.""" | ||
|
|
||
| import functools | ||
|
|
||
| from google.cloud.storage._experimental.asyncio.async_helpers import ASYNC_DEFAULT_TIMEOUT | ||
| from google.cloud.storage._experimental.asyncio.async_helpers import ASYNC_DEFAULT_RETRY | ||
| from google.cloud.storage._experimental.asyncio.async_helpers import AsyncHTTPIterator | ||
| from google.cloud.storage._experimental.asyncio.async_helpers import _do_nothing_page_start | ||
| from google.cloud.storage._opentelemetry_tracing import create_trace_span | ||
| from google.cloud.storage._experimental.asyncio.async_creds import AsyncCredsWrapper | ||
| from google.cloud.storage.abstracts.base_client import BaseClient | ||
| from google.cloud.storage._experimental.asyncio.async_connection import AsyncConnection | ||
| from google.cloud.storage._experimental.asyncio.utility.async_json_connection import ( | ||
| AsyncJSONConnection, | ||
| ) | ||
| from google.cloud.storage.abstracts import base_client | ||
|
|
||
| try: | ||
| from google.auth.aio.transport import sessions | ||
| AsyncSession = sessions.AsyncAuthorizedSession | ||
| _AIO_AVAILABLE = True | ||
| except ImportError: | ||
| _AIO_AVAILABLE = False | ||
|
|
||
| _marker = base_client.marker | ||
|
|
||
|
|
||
|
|
@@ -50,13 +38,6 @@ def __init__( | |
| *, | ||
| api_key=None, | ||
| ): | ||
| if not _AIO_AVAILABLE: | ||
| # Python 3.9 or less comes with an older version of google-auth library which doesn't support asyncio | ||
| raise ImportError( | ||
| "Failed to import 'google.auth.aio', Consider using a newer python version (>=3.10)" | ||
| " or newer version of google-auth library to mitigate this issue." | ||
| ) | ||
|
|
||
| if self._use_client_cert: | ||
| # google.auth.aio.transports.sessions.AsyncAuthorizedSession currently doesn't support configuring mTLS. | ||
| # In future, we can monkey patch the above, and do provide mTLS support, but that is not a priority | ||
|
|
@@ -70,169 +51,38 @@ def __init__( | |
| client_info=client_info, | ||
| client_options=client_options, | ||
| extra_headers=extra_headers, | ||
| api_key=api_key | ||
| api_key=api_key, | ||
| ) | ||
| self.credentials = AsyncCredsWrapper(self._credentials) # self._credential is synchronous. | ||
| self._connection = AsyncConnection(self, **self.connection_kw_args) # adapter for async communication | ||
| self._async_http_internal = _async_http | ||
| self._async_http_passed_by_user = (_async_http is not None) | ||
| self.credentials = AsyncCredsWrapper( | ||
| self._credentials | ||
| ) # self._credential is synchronous. | ||
| self._async_http = _async_http | ||
|
|
||
| @property | ||
| def async_http(self): | ||
| """Returns the existing asynchronous session, or create one if it does not exists.""" | ||
| if self._async_http_internal is None: | ||
| self._async_http_internal = AsyncSession(credentials=self.credentials) | ||
| return self._async_http_internal | ||
|
|
||
| async def close(self): | ||
| """Close the session, if it exists""" | ||
| if self._async_http_internal is not None and not self._async_http_passed_by_user: | ||
| await self._async_http_internal.close() | ||
| # We need both, as the same client can be used for multiple buckets. | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. for one client instance, the transport should be fixed right ? Are we planning to dynamically switch b/w json/grpc based on API endpoints ? |
||
| self._json_connection_internal = None | ||
| self._grpc_connection_internal = None | ||
|
|
||
| async def _get_resource( | ||
| self, | ||
| path, | ||
| query_params=None, | ||
| headers=None, | ||
| timeout=ASYNC_DEFAULT_TIMEOUT, | ||
| retry=ASYNC_DEFAULT_RETRY, | ||
| _target_object=None, | ||
| ): | ||
| """See super() class""" | ||
| return await self._connection.api_request( | ||
| method="GET", | ||
| path=path, | ||
| query_params=query_params, | ||
| headers=headers, | ||
| timeout=timeout, | ||
| retry=retry, | ||
| _target_object=_target_object, | ||
| ) | ||
| @property | ||
| def _grpc_connection(self): | ||
| raise NotImplementedError("Not yet Implemented.") | ||
|
|
||
| def _list_resource( | ||
| self, | ||
| path, | ||
| item_to_value, | ||
| page_token=None, | ||
| max_results=None, | ||
| extra_params=None, | ||
| page_start=_do_nothing_page_start, | ||
| page_size=None, | ||
| timeout=ASYNC_DEFAULT_TIMEOUT, | ||
| retry=ASYNC_DEFAULT_RETRY, | ||
| ): | ||
| """See super() class""" | ||
| kwargs = { | ||
| "method": "GET", | ||
| "path": path, | ||
| "timeout": timeout, | ||
| } | ||
| with create_trace_span( | ||
| name="Storage.AsyncClient._list_resource_returns_iterator", | ||
| client=self, | ||
| api_request=kwargs, | ||
| retry=retry, | ||
| ): | ||
| api_request = functools.partial( | ||
| self._connection.api_request, timeout=timeout, retry=retry | ||
| @property | ||
| def _json_connection(self): | ||
| if not self._json_connection_internal: | ||
| self._json_connection_internal = AsyncJSONConnection( | ||
| self, | ||
| _async_http=self._async_http, | ||
| credentials=self.credentials, | ||
| **self.connection_kw_args, | ||
| ) | ||
| return AsyncHTTPIterator( | ||
| client=self, | ||
| api_request=api_request, | ||
| path=path, | ||
| item_to_value=item_to_value, | ||
| page_token=page_token, | ||
| max_results=max_results, | ||
| extra_params=extra_params, | ||
| page_start=page_start, | ||
| page_size=page_size, | ||
| ) | ||
| return self._json_connection_internal | ||
|
|
||
| async def _patch_resource( | ||
| self, | ||
| path, | ||
| data, | ||
| query_params=None, | ||
| headers=None, | ||
| timeout=ASYNC_DEFAULT_TIMEOUT, | ||
| retry=None, | ||
| _target_object=None, | ||
| ): | ||
| """See super() class""" | ||
| return await self._connection.api_request( | ||
| method="PATCH", | ||
| path=path, | ||
| data=data, | ||
| query_params=query_params, | ||
| headers=headers, | ||
| timeout=timeout, | ||
| retry=retry, | ||
| _target_object=_target_object, | ||
| ) | ||
|
|
||
| async def _put_resource( | ||
| self, | ||
| path, | ||
| data, | ||
| query_params=None, | ||
| headers=None, | ||
| timeout=ASYNC_DEFAULT_TIMEOUT, | ||
| retry=None, | ||
| _target_object=None, | ||
| ): | ||
| """See super() class""" | ||
| return await self._connection.api_request( | ||
| method="PUT", | ||
| path=path, | ||
| data=data, | ||
| query_params=query_params, | ||
| headers=headers, | ||
| timeout=timeout, | ||
| retry=retry, | ||
| _target_object=_target_object, | ||
| ) | ||
|
|
||
| async def _post_resource( | ||
| self, | ||
| path, | ||
| data, | ||
| query_params=None, | ||
| headers=None, | ||
| timeout=ASYNC_DEFAULT_TIMEOUT, | ||
| retry=None, | ||
| _target_object=None, | ||
| ): | ||
| """See super() class""" | ||
| return await self._connection.api_request( | ||
| method="POST", | ||
| path=path, | ||
| data=data, | ||
| query_params=query_params, | ||
| headers=headers, | ||
| timeout=timeout, | ||
| retry=retry, | ||
| _target_object=_target_object, | ||
| ) | ||
| async def close(self): | ||
| if self._json_connection_internal: | ||
| await self._json_connection_internal.close() | ||
|
|
||
| async def _delete_resource( | ||
| self, | ||
| path, | ||
| query_params=None, | ||
| headers=None, | ||
| timeout=ASYNC_DEFAULT_TIMEOUT, | ||
| retry=ASYNC_DEFAULT_RETRY, | ||
| _target_object=None, | ||
| ): | ||
| """See super() class""" | ||
| return await self._connection.api_request( | ||
| method="DELETE", | ||
| path=path, | ||
| query_params=query_params, | ||
| headers=headers, | ||
| timeout=timeout, | ||
| retry=retry, | ||
| _target_object=_target_object, | ||
| ) | ||
| if self._grpc_connection_internal: | ||
| await self._grpc_connection_internal.close() | ||
|
|
||
| def bucket(self, bucket_name, user_project=None, generation=None): | ||
| """Factory constructor for bucket object. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: 2026