Skip to content

Commit 2e5d57c

Browse files
author
Jon Wayne Parrott
committed
Removing utils, putting depending code directly in samples
1 parent 7b5f96c commit 2e5d57c

File tree

10 files changed

+217
-142
lines changed

10 files changed

+217
-142
lines changed

‎bigquery/samples/async_query.py‎

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@
1313
#
1414
import argparse
1515
import json
16+
import time
1617
import uuid
1718

18-
from .utils import get_service, paging, poll_job
19+
from googleapiclient import discovery
20+
from oauth2client.client import GoogleCredentials
1921

2022

2123
# [START async_query]
22-
def async_query(service, project_id, query, batch=False, num_retries=5):
24+
def async_query(bigquery, project_id, query, batch=False, num_retries=5):
2325
# Generate a unique job_id so retries
2426
# don't accidentally duplicate query
2527
job_data = {
@@ -34,34 +36,67 @@ def async_query(service, project_id, query, batch=False, num_retries=5):
3436
}
3537
}
3638
}
37-
return service.jobs().insert(
39+
return bigquery.jobs().insert(
3840
projectId=project_id,
3941
body=job_data).execute(num_retries=num_retries)
4042
# [END async_query]
4143

4244

45+
# [START poll_job]
46+
def poll_job(bigquery, job):
47+
"""Waits for a job to complete."""
48+
49+
print('Waiting for job to finish...')
50+
51+
request = bigquery.jobs().get(
52+
projectId=job['jobReference']['projectId'],
53+
jobId=job['jobReference']['jobId'])
54+
55+
while True:
56+
result = request.execute(num_retries=2)
57+
58+
if result['status']['state'] == 'DONE':
59+
if 'errorResult' in result['status']:
60+
raise RuntimeError(result['status']['errorResult'])
61+
print('Job complete.')
62+
return
63+
64+
time.sleep(1)
65+
# [END poll_job]
66+
67+
4368
# [START run]
4469
def main(project_id, query_string, batch, num_retries, interval):
45-
service = get_service()
70+
# [START build_service]
71+
# Grab the application's default credentials from the environment.
72+
credentials = GoogleCredentials.get_application_default()
4673

47-
query_job = async_query(service,
48-
project_id,
49-
query_string,
50-
batch,
51-
num_retries)
74+
# Construct the service object for interacting with the BigQuery API.
75+
bigquery = discovery.build('bigquery', 'v2', credentials=credentials)
76+
# [END build_service]
5277

53-
poll_job(service,
54-
query_job['jobReference']['projectId'],
55-
query_job['jobReference']['jobId'],
56-
interval,
57-
num_retries)
78+
# Submit the job and wait for it to complete.
79+
query_job = async_query(
80+
bigquery,
81+
project_id,
82+
query_string,
83+
batch,
84+
num_retries)
5885

59-
for page in paging(service,
60-
service.jobs().getQueryResults,
61-
num_retries=num_retries,
62-
**query_job['jobReference']):
86+
poll_job(bigquery, query_job)
87+
88+
# Page through the result set and print all results.
89+
page_token = None
90+
while True:
91+
page = bigquery.jobs().getQueryResults(
92+
pageToken=page_token,
93+
**query_job['jobReference']).execute(num_retries=2)
6394

6495
print(json.dumps(page['rows']))
96+
97+
page_token = page.get('pageToken')
98+
if not page_token:
99+
break
65100
# [END run]
66101

67102

‎bigquery/samples/export_data_to_cloud_storage.py‎

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,23 @@
1212
# limitations under the License.
1313
#
1414
import argparse
15+
import time
1516
import uuid
1617

17-
from .utils import get_service, poll_job
18+
from googleapiclient import discovery
19+
from oauth2client.client import GoogleCredentials
1820

1921

2022
# [START export_table]
21-
def export_table(service, cloud_storage_path,
23+
def export_table(bigquery, cloud_storage_path,
2224
project_id, dataset_id, table_id,
2325
export_format="CSV",
2426
num_retries=5):
2527
"""
2628
Starts an export job
2729
2830
Args:
29-
service: initialized and authorized bigquery
31+
bigquery: initialized and authorized bigquery
3032
google-api-client object.
3133
cloud_storage_path: fully qualified
3234
path to a Google Cloud Storage location.
@@ -56,26 +58,55 @@ def export_table(service, cloud_storage_path,
5658
}
5759
}
5860
}
59-
return service.jobs().insert(
61+
return bigquery.jobs().insert(
6062
projectId=project_id,
6163
body=job_data).execute(num_retries=num_retries)
6264
# [END export_table]
6365

6466

67+
# [START poll_job]
68+
def poll_job(bigquery, job):
69+
"""Waits for a job to complete."""
70+
71+
print('Waiting for job to finish...')
72+
73+
request = bigquery.jobs().get(
74+
projectId=job['jobReference']['projectId'],
75+
jobId=job['jobReference']['jobId'])
76+
77+
while True:
78+
result = request.execute(num_retries=2)
79+
80+
if result['status']['state'] == 'DONE':
81+
if 'errorResult' in result['status']:
82+
raise RuntimeError(result['status']['errorResult'])
83+
print('Job complete.')
84+
return
85+
86+
time.sleep(1)
87+
# [END poll_job]
88+
89+
6590
# [START run]
6691
def main(cloud_storage_path, project_id, dataset_id, table_id,
6792
num_retries, interval, export_format="CSV"):
93+
# [START build_service]
94+
# Grab the application's default credentials from the environment.
95+
credentials = GoogleCredentials.get_application_default()
96+
97+
# Construct the service object for interacting with the BigQuery API.
98+
bigquery = discovery.build('bigquery', 'v2', credentials=credentials)
99+
# [END build_service]
68100

69-
bigquery = get_service()
70-
resource = export_table(bigquery, cloud_storage_path,
71-
project_id, dataset_id, table_id,
72-
num_retries=num_retries,
73-
export_format=export_format)
74-
poll_job(bigquery,
75-
resource['jobReference']['projectId'],
76-
resource['jobReference']['jobId'],
77-
interval,
78-
num_retries)
101+
job = export_table(
102+
bigquery,
103+
cloud_storage_path,
104+
project_id,
105+
dataset_id,
106+
table_id,
107+
num_retries=num_retries,
108+
export_format=export_format)
109+
poll_job(bigquery, job)
79110
# [END run]
80111

81112

‎bigquery/samples/list_datasets_projects.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def list_datasets(service, project):
6969

7070
except HTTPError as err:
7171
print('Error in list_datasets: %s' % err.content)
72+
raise err
7273
# [END list_datasets]
7374

7475

@@ -84,6 +85,7 @@ def list_projects(service):
8485

8586
except HTTPError as err:
8687
print('Error in list_projects: %s' % err.content)
88+
raise err
8789
# [END list_projects]
8890

8991

‎bigquery/samples/load_data_by_post.py‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@
1313
#
1414
import argparse
1515
import json
16+
import time
1617

18+
from googleapiclient import discovery
1719
import httplib2
1820
from oauth2client.client import GoogleCredentials
19-
from .utils import get_service, poll_job
2021

2122

2223
# [START make_post]
23-
def make_post(http, schema, data, projectId, datasetId, tableId):
24+
def make_post(http, schema, data, project_id, dataset_id, table_id):
2425
"""
2526
Creates an http POST request for loading data into
2627
a bigquery table
@@ -34,7 +35,7 @@ def make_post(http, schema, data, projectId, datasetId, tableId):
3435
Returns: an http.request object
3536
"""
3637
url = ('https://www.googleapis.com/upload/bigquery/v2/projects/' +
37-
projectId + '/jobs')
38+
project_id + '/jobs')
3839
# Create the body of the request, separated by a boundary of xxx
3940
resource = ('--xxx\n' +
4041
'Content-Type: application/json; charset=UTF-8\n' + '\n' +
@@ -45,9 +46,9 @@ def make_post(http, schema, data, projectId, datasetId, tableId):
4546
' "fields": ' + str(schema) + '\n' +
4647
' },\n' +
4748
' "destinationTable": {\n' +
48-
' "projectId": "' + projectId + '",\n' +
49-
' "datasetId": "' + datasetId + '",\n' +
50-
' "tableId": "' + tableId + '"\n' +
49+
' "projectId": "' + project_id + '",\n' +
50+
' "datasetId": "' + dataset_id + '",\n' +
51+
' "tableId": "' + table_id + '"\n' +
5152
' }\n' +
5253
' }\n' +
5354
' }\n' +
@@ -70,10 +71,34 @@ def make_post(http, schema, data, projectId, datasetId, tableId):
7071
# [END make_post]
7172

7273

74+
# [START poll_job]
75+
def poll_job(bigquery, job):
76+
"""Waits for a job to complete."""
77+
78+
print('Waiting for job to finish...')
79+
80+
request = bigquery.jobs().get(
81+
projectId=job['jobReference']['projectId'],
82+
jobId=job['jobReference']['jobId'])
83+
84+
while True:
85+
result = request.execute(num_retries=2)
86+
87+
if result['status']['state'] == 'DONE':
88+
if 'errorResult' in result['status']:
89+
raise RuntimeError(result['status']['errorResult'])
90+
print('Job complete.')
91+
return
92+
93+
time.sleep(1)
94+
# [END poll_job]
95+
96+
7397
# [START main]
7498
def main(project_id, dataset_id, table_name, schema_path, data_path):
7599
credentials = GoogleCredentials.get_application_default()
76100
http = credentials.authorize(httplib2.Http())
101+
bigquery = discovery.build('bigquery', 'v2', credentials=credentials)
77102

78103
with open(schema_path, 'r') as schema_file:
79104
schema = schema_file.read()
@@ -90,9 +115,8 @@ def main(project_id, dataset_id, table_name, schema_path, data_path):
90115
table_name)
91116

92117
if resp.status == 200:
93-
job_resource = json.loads(content)
94-
service = get_service()
95-
poll_job(service, **job_resource['jobReference'])
118+
job = json.loads(content)
119+
poll_job(bigquery, job)
96120
print("Success!")
97121
else:
98122
print("Http error code: {}".format(resp.status))

‎bigquery/samples/load_data_from_csv.py‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,21 @@
1313
#
1414
import argparse
1515
import json
16+
import time
1617
import uuid
1718

18-
from .utils import get_service, poll_job
19+
from googleapiclient import discovery
20+
from oauth2client.client import GoogleCredentials
1921

2022

2123
# [START load_table]
22-
def load_table(service, project_id, dataset_id, table_name, source_schema,
24+
def load_table(bigquery, project_id, dataset_id, table_name, source_schema,
2325
source_path, num_retries=5):
2426
"""
2527
Starts a job to load a bigquery table from CSV
2628
2729
Args:
28-
service: an initialized and authorized bigquery
30+
bigquery: an initialized and authorized bigquery client
2931
google-api-client object
3032
source_schema: a valid bigquery schema,
3133
see https://cloud.google.com/bigquery/docs/reference/v2/tables
@@ -58,34 +60,59 @@ def load_table(service, project_id, dataset_id, table_name, source_schema,
5860
}
5961
}
6062

61-
return service.jobs().insert(
63+
return bigquery.jobs().insert(
6264
projectId=project_id,
6365
body=job_data).execute(num_retries=num_retries)
6466
# [END load_table]
6567

6668

69+
# [START poll_job]
70+
def poll_job(bigquery, job):
71+
"""Waits for a job to complete."""
72+
73+
print('Waiting for job to finish...')
74+
75+
request = bigquery.jobs().get(
76+
projectId=job['jobReference']['projectId'],
77+
jobId=job['jobReference']['jobId'])
78+
79+
while True:
80+
result = request.execute(num_retries=2)
81+
82+
if result['status']['state'] == 'DONE':
83+
if 'errorResult' in result['status']:
84+
raise RuntimeError(result['status']['errorResult'])
85+
print('Job complete.')
86+
return
87+
88+
time.sleep(1)
89+
# [END poll_job]
90+
91+
6792
# [START run]
6893
def main(project_id, dataset_id, table_name, schema_file, data_path,
6994
poll_interval, num_retries):
70-
service = get_service()
95+
# [START build_service]
96+
# Grab the application's default credentials from the environment.
97+
credentials = GoogleCredentials.get_application_default()
98+
99+
# Construct the service object for interacting with the BigQuery API.
100+
bigquery = discovery.build('bigquery', 'v2', credentials=credentials)
101+
# [END build_service]
71102

72103
with open(schema_file, 'r') as f:
73104
schema = json.load(f)
74105

75106
job = load_table(
76-
service,
107+
bigquery,
77108
project_id,
78109
dataset_id,
79110
table_name,
80111
schema,
81112
data_path,
82113
num_retries)
83114

84-
poll_job(service,
85-
job['jobReference']['projectId'],
86-
job['jobReference']['jobId'],
87-
poll_interval,
88-
num_retries)
115+
poll_job(bigquery, job)
89116
# [END run]
90117

91118

0 commit comments

Comments
 (0)