Skip to content

#825: Allow passing explicit connection to 'Bucket.{list_blobs,make public}' #856

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tseaver merged 4 commits into from
May 11, 2015
Merged

#825: Allow passing explicit connection to 'Bucket.{list_blobs,make public}' #856

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
245a090
Document 'extra_params'.
tseaver May 1, 2015
f97c62c
#825: Allow passing explicit connection to '_BlobIterator.__init__'.
tseaver May 1, 2015
5ed6c7e
#825: Allow passing explicit connection to 'Bucket.{list_blobs,make_…
tseaver May 1, 2015
8d6ef05
Iterator uses implicit connection, rather than 'bucket.connection'.
tseaver May 7, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 34 additions & 11 deletions gcloud/storage/bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,20 @@ class _BlobIterator(Iterator):

:type bucket: :class:`gcloud.storage.bucket.Bucket`
:param bucket: The bucket from which to list blobs.

:type extra_params: dict or None
:param extra_params: Extra query string parameters for the API call.

:type connection: :class:`gcloud.storage.connection.Connection`
:param connection: The connection to use when sending requests. Defaults
to the bucket's connection
"""
def __init__(self, bucket, extra_params=None):
def __init__(self, bucket, extra_params=None, connection=None):
connection = _require_connection(connection)
self.bucket = bucket
self.prefixes = ()
super(_BlobIterator, self).__init__(
connection=bucket.connection, path=bucket.path + '/o',
connection=connection, path=bucket.path + '/o',
extra_params=extra_params)

def get_items_from_response(self, response):
Expand Down Expand Up @@ -257,7 +265,7 @@ def get_blob(self, blob_name, connection=None):

def list_blobs(self, max_results=None, page_token=None, prefix=None,
delimiter=None, versions=None,
projection='noAcl', fields=None):
projection='noAcl', fields=None, connection=None):
"""Return an iterator used to find blobs in the bucket.

:type max_results: integer or ``NoneType``
Expand Down Expand Up @@ -289,6 +297,11 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None,
and the language of each blob returned:
'items/contentLanguage,nextPageToken'

:type connection: :class:`gcloud.storage.connection.Connection` or
``NoneType``
:param connection: Optional. The connection to use when sending
requests. If not provided, falls back to default.

:rtype: :class:`_BlobIterator`.
:returns: An iterator of blobs.
"""
Expand All @@ -311,7 +324,8 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None,
if fields is not None:
extra_params['fields'] = fields

result = self._iterator_class(self, extra_params=extra_params)
result = self._iterator_class(
self, extra_params=extra_params, connection=connection)
# Page token must be handled specially since the base `Iterator`
# class has it as a reserved property.
if page_token is not None:
Expand Down Expand Up @@ -348,7 +362,8 @@ def delete(self, force=False, connection=None):
connection = _require_connection(connection)
if force:
blobs = list(self.list_blobs(
max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1))
max_results=self._MAX_OBJECTS_FOR_BUCKET_DELETE + 1,
connection=connection))
if len(blobs) > self._MAX_OBJECTS_FOR_BUCKET_DELETE:
message = (
'Refusing to delete bucket with more than '
Expand Down Expand Up @@ -844,7 +859,7 @@ def disable_website(self):
"""
return self.configure_website(None, None)

def make_public(self, recursive=False, future=False):
def make_public(self, recursive=False, future=False, connection=None):
"""Make a bucket public.

:type recursive: boolean
Expand All @@ -854,18 +869,26 @@ def make_public(self, recursive=False, future=False):
:type future: boolean
:param future: If True, this will make all objects created in the
future public as well.

:type connection: :class:`gcloud.storage.connection.Connection` or
``NoneType``
:param connection: Optional. The connection to use when sending
requests. If not provided, falls back to default.
"""
connection = _require_connection(connection)

self.acl.all().grant_read()
self.acl.save()
self.acl.save(connection=connection)

if future:
doa = self.default_object_acl
if not doa.loaded:
doa.reload()
doa.reload(connection=connection)
doa.all().grant_read()
doa.save()
doa.save(connection=connection)

if recursive:
for blob in self:
for blob in self.list_blobs(projection='full',
connection=connection):
blob.acl.all().grant_read()
blob.save_acl()
blob.acl.save(connection=connection)

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

104 changes: 72 additions & 32 deletions gcloud/storage/test_bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,23 @@ def _getTargetClass(self):
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)

def test_ctor(self):
def test_ctor_w_implicit_connection(self):
from gcloud.storage._testing import _monkey_defaults
connection = _Connection()
bucket = _Bucket(connection)
iterator = self._makeOne(bucket)
bucket = _Bucket(None)
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
self.assertTrue(iterator.bucket is bucket)
self.assertTrue(iterator.connection is connection)
self.assertEqual(iterator.path, '%s/o' % bucket.path)
self.assertEqual(iterator.page_number, 0)
self.assertEqual(iterator.next_page_token, None)
self.assertEqual(iterator.prefixes, ())

def test_ctor_w_explicit_connection(self):
connection = _Connection()
bucket = _Bucket(None)
iterator = self._makeOne(bucket, connection=connection)
self.assertTrue(iterator.bucket is bucket)
self.assertTrue(iterator.connection is connection)
self.assertEqual(iterator.path, '%s/o' % bucket.path)
Expand All @@ -38,20 +51,25 @@ def test_ctor(self):
self.assertEqual(iterator.prefixes, ())

def test_get_items_from_response_empty(self):
from gcloud.storage._testing import _monkey_defaults
connection = _Connection()
bucket = _Bucket(connection)
iterator = self._makeOne(bucket)
self.assertEqual(list(iterator.get_items_from_response({})), [])
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response({}))
self.assertEqual(blobs, [])
self.assertEqual(iterator.prefixes, ())

def test_get_items_from_response_non_empty(self):
from gcloud.storage.blob import Blob
from gcloud.storage._testing import _monkey_defaults
BLOB_NAME = 'blob-name'
response = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']}
connection = _Connection()
bucket = _Bucket(connection)
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response(response))
with _monkey_defaults(connection=connection):
iterator = self._makeOne(bucket)
blobs = list(iterator.get_items_from_response(response))
self.assertEqual(len(blobs), 1)
blob = blobs[0]
self.assertTrue(isinstance(blob, Blob))
Expand Down Expand Up @@ -93,22 +111,26 @@ def test_ctor_explicit(self):
self.assertTrue(bucket._default_object_acl.bucket is bucket)

def test___iter___empty(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, connection)
blobs = list(bucket)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
blobs = list(bucket)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test___iter___non_empty(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
BLOB_NAME = 'blob-name'
connection = _Connection({'items': [{'name': BLOB_NAME}]})
bucket = self._makeOne(NAME, connection)
blobs = list(bucket)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
blobs = list(bucket)
blob, = blobs
self.assertTrue(blob.bucket is bucket)
self.assertEqual(blob.name, BLOB_NAME)
Expand Down Expand Up @@ -279,18 +301,21 @@ def test_get_blob_hit(self):
self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME))

def test_list_blobs_defaults(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, connection)
iterator = bucket.list_blobs()
blobs = list(iterator)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
iterator = bucket.list_blobs()
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test_list_blobs_explicit(self):
from gcloud.storage._testing import _monkey_defaults
NAME = 'name'
MAX_RESULTS = 10
PAGE_TOKEN = 'ABCD'
Expand All @@ -309,23 +334,36 @@ def test_list_blobs_explicit(self):
'fields': FIELDS,
}
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, connection)
iterator = bucket.list_blobs(
max_results=MAX_RESULTS,
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
)
blobs = list(iterator)
bucket = self._makeOne(NAME)
with _monkey_defaults(connection=connection):
iterator = bucket.list_blobs(
max_results=MAX_RESULTS,
page_token=PAGE_TOKEN,
prefix=PREFIX,
delimiter=DELIMITER,
versions=VERSIONS,
projection=PROJECTION,
fields=FIELDS,
)
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], EXPECTED)

def test_list_blobs_w_explicit_connection(self):
NAME = 'name'
connection = _Connection({'items': []})
bucket = self._makeOne(NAME, None)
iterator = bucket.list_blobs(connection=connection)
blobs = list(iterator)
self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw['method'], 'GET')
self.assertEqual(kw['path'], '/b/%s/o' % NAME)
self.assertEqual(kw['query_params'], {'projection': 'noAcl'})

def test_delete_default_miss(self):
from gcloud.exceptions import NotFound
NAME = 'name'
Expand Down Expand Up @@ -895,7 +933,7 @@ def test_make_public_defaults(self):
permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}]
after = {'acl': permissive, 'defaultObjectAcl': []}
connection = _Connection(after)
bucket = self._makeOne(NAME, connection)
bucket = self._makeOne(NAME, None)
bucket.acl.loaded = True
bucket.default_object_acl.loaded = True
with _monkey_defaults(connection=connection):
Expand Down Expand Up @@ -924,7 +962,7 @@ def _make_public_w_future_helper(self, default_object_acl_loaded=True):
# We return the same value for default_object_acl.reload()
# to consume.
connection = _Connection(after1, after1, after2)
bucket = self._makeOne(NAME, connection)
bucket = self._makeOne(NAME, None)
bucket.acl.loaded = True
bucket.default_object_acl.loaded = default_object_acl_loaded
with _monkey_defaults(connection=connection):
Expand Down Expand Up @@ -969,14 +1007,16 @@ def __init__(self, bucket, name):
def acl(self):
return self

# Faux ACL methods
def all(self):
return self

def grant_read(self):
self._granted = True

def save_acl(self):
_saved.append((self._bucket, self._name, self._granted))
def save(self, connection=None):
_saved.append(
(self._bucket, self._name, self._granted, connection))

class _Iterator(_BlobIterator):
def get_items_from_response(self, response):
Expand All @@ -988,15 +1028,15 @@ def get_items_from_response(self, response):
permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}]
after = {'acl': permissive, 'defaultObjectAcl': []}
connection = _Connection(after, {'items': [{'name': BLOB_NAME}]})
bucket = self._makeOne(NAME, connection)
bucket = self._makeOne(NAME, None)
bucket.acl.loaded = True
bucket.default_object_acl.loaded = True
bucket._iterator_class = _Iterator
with _monkey_defaults(connection=connection):
bucket.make_public(recursive=True)
self.assertEqual(list(bucket.acl), permissive)
self.assertEqual(list(bucket.default_object_acl), [])
self.assertEqual(_saved, [(bucket, BLOB_NAME, True)])
self.assertEqual(_saved, [(bucket, BLOB_NAME, True, connection)])
kw = connection._requested
self.assertEqual(len(kw), 2)
self.assertEqual(kw[0]['method'], 'PATCH')
Expand All @@ -1005,7 +1045,7 @@ def get_items_from_response(self, response):
self.assertEqual(kw[0]['query_params'], {'projection': 'full'})
self.assertEqual(kw[1]['method'], 'GET')
self.assertEqual(kw[1]['path'], '/b/%s/o' % NAME)
self.assertEqual(kw[1]['query_params'], {'projection': 'noAcl'})
self.assertEqual(kw[1]['query_params'], {'projection': 'full'})


class _Connection(object):
Expand Down