Skip to content

Commit 079b58a

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: add open lineage support (googleapis#11987)
- [ ] Regenerate this pull request now. BEGIN_COMMIT_OVERRIDE fix: change `start_time` in message `.google.cloud.datacatalog.lineage.v1.LineageEvent` to `required` as intended by api feat: add open lineage support END_COMMIT_OVERRIDE PiperOrigin-RevId: 579762272 Source-Link: googleapis/googleapis@58878bd Source-Link: https://github.com/googleapis/googleapis-gen/commit/96a4d735b92f202fbdd2c62da2e21e63216bfa88 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRhdGFjYXRhbG9nLWxpbmVhZ2UvLk93bEJvdC55YW1sIiwiaCI6Ijk2YTRkNzM1YjkyZjIwMmZiZGQyYzYyZGEyZTIxZTYzMjE2YmZhODgifQ== --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent bf5ee9f commit 079b58a

19 files changed

Lines changed: 1757 additions & 210 deletions

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@
4949
Process,
5050
ProcessLinkInfo,
5151
ProcessLinks,
52+
ProcessOpenLineageRunEventRequest,
53+
ProcessOpenLineageRunEventResponse,
5254
Run,
5355
SearchLinksRequest,
5456
SearchLinksResponse,
@@ -85,6 +87,8 @@
8587
"Process",
8688
"ProcessLinkInfo",
8789
"ProcessLinks",
90+
"ProcessOpenLineageRunEventRequest",
91+
"ProcessOpenLineageRunEventResponse",
8892
"Run",
8993
"SearchLinksRequest",
9094
"SearchLinksResponse",

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage/gapic_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
#
16-
__version__ = "0.3.0" # {x-release-please-version}
16+
__version__ = "0.0.0" # {x-release-please-version}

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage_v1/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@
4646
Process,
4747
ProcessLinkInfo,
4848
ProcessLinks,
49+
ProcessOpenLineageRunEventRequest,
50+
ProcessOpenLineageRunEventResponse,
4951
Run,
5052
SearchLinksRequest,
5153
SearchLinksResponse,
@@ -82,6 +84,8 @@
8284
"Process",
8385
"ProcessLinkInfo",
8486
"ProcessLinks",
87+
"ProcessOpenLineageRunEventRequest",
88+
"ProcessOpenLineageRunEventResponse",
8589
"Run",
8690
"SearchLinksRequest",
8791
"SearchLinksResponse",

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage_v1/gapic_metadata.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@
7575
"list_runs"
7676
]
7777
},
78+
"ProcessOpenLineageRunEvent": {
79+
"methods": [
80+
"process_open_lineage_run_event"
81+
]
82+
},
7883
"SearchLinks": {
7984
"methods": [
8085
"search_links"
@@ -160,6 +165,11 @@
160165
"list_runs"
161166
]
162167
},
168+
"ProcessOpenLineageRunEvent": {
169+
"methods": [
170+
"process_open_lineage_run_event"
171+
]
172+
},
163173
"SearchLinks": {
164174
"methods": [
165175
"search_links"
@@ -245,6 +255,11 @@
245255
"list_runs"
246256
]
247257
},
258+
"ProcessOpenLineageRunEvent": {
259+
"methods": [
260+
"process_open_lineage_run_event"
261+
]
262+
},
248263
"SearchLinks": {
249264
"methods": [
250265
"search_links"

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage_v1/gapic_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
#
16-
__version__ = "0.3.0" # {x-release-please-version}
16+
__version__ = "0.0.0" # {x-release-please-version}

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage_v1/services/lineage/async_client.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
from google.longrunning import operations_pb2 # type: ignore
4848
from google.protobuf import empty_pb2 # type: ignore
4949
from google.protobuf import field_mask_pb2 # type: ignore
50+
from google.protobuf import struct_pb2 # type: ignore
5051
from google.protobuf import timestamp_pb2 # type: ignore
5152

5253
from google.cloud.datacatalog_lineage_v1.services.lineage import pagers
@@ -222,6 +223,127 @@ def __init__(
222223
client_info=client_info,
223224
)
224225

226+
async def process_open_lineage_run_event(
227+
self,
228+
request: Optional[
229+
Union[lineage.ProcessOpenLineageRunEventRequest, dict]
230+
] = None,
231+
*,
232+
parent: Optional[str] = None,
233+
open_lineage: Optional[struct_pb2.Struct] = None,
234+
retry: OptionalRetry = gapic_v1.method.DEFAULT,
235+
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
236+
metadata: Sequence[Tuple[str, str]] = (),
237+
) -> lineage.ProcessOpenLineageRunEventResponse:
238+
r"""Creates new lineage events together with their
239+
parents: process and run. Updates the process and run if
240+
they already exist. Mapped from Open Lineage
241+
specification:
242+
243+
https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
244+
245+
.. code-block:: python
246+
247+
# This snippet has been automatically generated and should be regarded as a
248+
# code template only.
249+
# It will require modifications to work:
250+
# - It may require correct/in-range values for request initialization.
251+
# - It may require specifying regional endpoints when creating the service
252+
# client as shown in:
253+
# https://googleapis.dev/python/google-api-core/latest/client_options.html
254+
from google.cloud import datacatalog_lineage_v1
255+
256+
async def sample_process_open_lineage_run_event():
257+
# Create a client
258+
client = datacatalog_lineage_v1.LineageAsyncClient()
259+
260+
# Initialize request argument(s)
261+
request = datacatalog_lineage_v1.ProcessOpenLineageRunEventRequest(
262+
parent="parent_value",
263+
)
264+
265+
# Make the request
266+
response = await client.process_open_lineage_run_event(request=request)
267+
268+
# Handle the response
269+
print(response)
270+
271+
Args:
272+
request (Optional[Union[google.cloud.datacatalog_lineage_v1.types.ProcessOpenLineageRunEventRequest, dict]]):
273+
The request object. Request message for
274+
[ProcessOpenLineageRunEvent][google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEvent].
275+
parent (:class:`str`):
276+
Required. The name of the project and
277+
its location that should own the
278+
process, run, and lineage event.
279+
280+
This corresponds to the ``parent`` field
281+
on the ``request`` instance; if ``request`` is provided, this
282+
should not be set.
283+
open_lineage (:class:`google.protobuf.struct_pb2.Struct`):
284+
Required. OpenLineage message
285+
following OpenLineage format:
286+
https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json
287+
288+
This corresponds to the ``open_lineage`` field
289+
on the ``request`` instance; if ``request`` is provided, this
290+
should not be set.
291+
retry (google.api_core.retry.Retry): Designation of what errors, if any,
292+
should be retried.
293+
timeout (float): The timeout for this request.
294+
metadata (Sequence[Tuple[str, str]]): Strings which should be
295+
sent along with the request as metadata.
296+
297+
Returns:
298+
google.cloud.datacatalog_lineage_v1.types.ProcessOpenLineageRunEventResponse:
299+
Response message for
300+
[ProcessOpenLineageRunEvent][google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEvent].
301+
302+
"""
303+
# Create or coerce a protobuf request object.
304+
# Quick check: If we got a request object, we should *not* have
305+
# gotten any keyword arguments that map to the request.
306+
has_flattened_params = any([parent, open_lineage])
307+
if request is not None and has_flattened_params:
308+
raise ValueError(
309+
"If the `request` argument is set, then none of "
310+
"the individual field arguments should be set."
311+
)
312+
313+
request = lineage.ProcessOpenLineageRunEventRequest(request)
314+
315+
# If we have keyword arguments corresponding to fields on the
316+
# request, apply these.
317+
if parent is not None:
318+
request.parent = parent
319+
if open_lineage is not None:
320+
request.open_lineage = open_lineage
321+
322+
# Wrap the RPC method; this adds retry and timeout information,
323+
# and friendly error handling.
324+
rpc = gapic_v1.method_async.wrap_method(
325+
self._client._transport.process_open_lineage_run_event,
326+
default_timeout=None,
327+
client_info=DEFAULT_CLIENT_INFO,
328+
)
329+
330+
# Certain fields should be provided within the metadata header;
331+
# add these here.
332+
metadata = tuple(metadata) + (
333+
gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)),
334+
)
335+
336+
# Send the request.
337+
response = await rpc(
338+
request,
339+
retry=retry,
340+
timeout=timeout,
341+
metadata=metadata,
342+
)
343+
344+
# Done; return the response.
345+
return response
346+
225347
async def create_process(
226348
self,
227349
request: Optional[Union[lineage.CreateProcessRequest, dict]] = None,

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage_v1/services/lineage/client.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
from google.longrunning import operations_pb2 # type: ignore
5252
from google.protobuf import empty_pb2 # type: ignore
5353
from google.protobuf import field_mask_pb2 # type: ignore
54+
from google.protobuf import struct_pb2 # type: ignore
5455
from google.protobuf import timestamp_pb2 # type: ignore
5556

5657
from google.cloud.datacatalog_lineage_v1.services.lineage import pagers
@@ -503,6 +504,129 @@ def __init__(
503504
api_audience=client_options.api_audience,
504505
)
505506

507+
def process_open_lineage_run_event(
508+
self,
509+
request: Optional[
510+
Union[lineage.ProcessOpenLineageRunEventRequest, dict]
511+
] = None,
512+
*,
513+
parent: Optional[str] = None,
514+
open_lineage: Optional[struct_pb2.Struct] = None,
515+
retry: OptionalRetry = gapic_v1.method.DEFAULT,
516+
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
517+
metadata: Sequence[Tuple[str, str]] = (),
518+
) -> lineage.ProcessOpenLineageRunEventResponse:
519+
r"""Creates new lineage events together with their
520+
parents: process and run. Updates the process and run if
521+
they already exist. Mapped from Open Lineage
522+
specification:
523+
524+
https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
525+
526+
.. code-block:: python
527+
528+
# This snippet has been automatically generated and should be regarded as a
529+
# code template only.
530+
# It will require modifications to work:
531+
# - It may require correct/in-range values for request initialization.
532+
# - It may require specifying regional endpoints when creating the service
533+
# client as shown in:
534+
# https://googleapis.dev/python/google-api-core/latest/client_options.html
535+
from google.cloud import datacatalog_lineage_v1
536+
537+
def sample_process_open_lineage_run_event():
538+
# Create a client
539+
client = datacatalog_lineage_v1.LineageClient()
540+
541+
# Initialize request argument(s)
542+
request = datacatalog_lineage_v1.ProcessOpenLineageRunEventRequest(
543+
parent="parent_value",
544+
)
545+
546+
# Make the request
547+
response = client.process_open_lineage_run_event(request=request)
548+
549+
# Handle the response
550+
print(response)
551+
552+
Args:
553+
request (Union[google.cloud.datacatalog_lineage_v1.types.ProcessOpenLineageRunEventRequest, dict]):
554+
The request object. Request message for
555+
[ProcessOpenLineageRunEvent][google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEvent].
556+
parent (str):
557+
Required. The name of the project and
558+
its location that should own the
559+
process, run, and lineage event.
560+
561+
This corresponds to the ``parent`` field
562+
on the ``request`` instance; if ``request`` is provided, this
563+
should not be set.
564+
open_lineage (google.protobuf.struct_pb2.Struct):
565+
Required. OpenLineage message
566+
following OpenLineage format:
567+
https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json
568+
569+
This corresponds to the ``open_lineage`` field
570+
on the ``request`` instance; if ``request`` is provided, this
571+
should not be set.
572+
retry (google.api_core.retry.Retry): Designation of what errors, if any,
573+
should be retried.
574+
timeout (float): The timeout for this request.
575+
metadata (Sequence[Tuple[str, str]]): Strings which should be
576+
sent along with the request as metadata.
577+
578+
Returns:
579+
google.cloud.datacatalog_lineage_v1.types.ProcessOpenLineageRunEventResponse:
580+
Response message for
581+
[ProcessOpenLineageRunEvent][google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEvent].
582+
583+
"""
584+
# Create or coerce a protobuf request object.
585+
# Quick check: If we got a request object, we should *not* have
586+
# gotten any keyword arguments that map to the request.
587+
has_flattened_params = any([parent, open_lineage])
588+
if request is not None and has_flattened_params:
589+
raise ValueError(
590+
"If the `request` argument is set, then none of "
591+
"the individual field arguments should be set."
592+
)
593+
594+
# Minor optimization to avoid making a copy if the user passes
595+
# in a lineage.ProcessOpenLineageRunEventRequest.
596+
# There's no risk of modifying the input as we've already verified
597+
# there are no flattened fields.
598+
if not isinstance(request, lineage.ProcessOpenLineageRunEventRequest):
599+
request = lineage.ProcessOpenLineageRunEventRequest(request)
600+
# If we have keyword arguments corresponding to fields on the
601+
# request, apply these.
602+
if parent is not None:
603+
request.parent = parent
604+
if open_lineage is not None:
605+
request.open_lineage = open_lineage
606+
607+
# Wrap the RPC method; this adds retry and timeout information,
608+
# and friendly error handling.
609+
rpc = self._transport._wrapped_methods[
610+
self._transport.process_open_lineage_run_event
611+
]
612+
613+
# Certain fields should be provided within the metadata header;
614+
# add these here.
615+
metadata = tuple(metadata) + (
616+
gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)),
617+
)
618+
619+
# Send the request.
620+
response = rpc(
621+
request,
622+
retry=retry,
623+
timeout=timeout,
624+
metadata=metadata,
625+
)
626+
627+
# Done; return the response.
628+
return response
629+
506630
def create_process(
507631
self,
508632
request: Optional[Union[lineage.CreateProcessRequest, dict]] = None,

‎packages/google-cloud-datacatalog-lineage/google/cloud/datacatalog_lineage_v1/services/lineage/transports/base.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ def __init__(
124124
def _prep_wrapped_messages(self, client_info):
125125
# Precompute the wrapped methods.
126126
self._wrapped_methods = {
127+
self.process_open_lineage_run_event: gapic_v1.method.wrap_method(
128+
self.process_open_lineage_run_event,
129+
default_timeout=None,
130+
client_info=client_info,
131+
),
127132
self.create_process: gapic_v1.method.wrap_method(
128133
self.create_process,
129134
default_timeout=None,
@@ -220,6 +225,18 @@ def operations_client(self):
220225
"""Return the client designed to process long-running operations."""
221226
raise NotImplementedError()
222227

228+
@property
229+
def process_open_lineage_run_event(
230+
self,
231+
) -> Callable[
232+
[lineage.ProcessOpenLineageRunEventRequest],
233+
Union[
234+
lineage.ProcessOpenLineageRunEventResponse,
235+
Awaitable[lineage.ProcessOpenLineageRunEventResponse],
236+
],
237+
]:
238+
raise NotImplementedError()
239+
223240
@property
224241
def create_process(
225242
self,

0 commit comments

Comments
 (0)