Merge pull request #932 from dhermes/fix-920

Making _BlobIterator accumulate prefixes.

Author: dhermes

gcloud/storage/bucket.py

@@ -52,7 +52,8 @@ class _BlobIterator(Iterator):
     def __init__(self, bucket, extra_params=None, connection=None):
         connection = _require_connection(connection)
         self.bucket = bucket
-        self.prefixes = ()
+        self.prefixes = set()
+        self._current_prefixes = None
         super(_BlobIterator, self).__init__(
             connection=connection, path=bucket.path + '/o',
             extra_params=extra_params)
@@ -63,7 +64,8 @@ def get_items_from_response(self, response):
         :type response: dict
         :param response: The JSON API response for a page of blobs.
         """
-        self.prefixes = tuple(response.get('prefixes', ()))
+        self._current_prefixes = tuple(response.get('prefixes', ()))
+        self.prefixes.update(self._current_prefixes)
         for item in response.get('items', []):
             name = item.get('name')
             blob = Blob(name, bucket=self.bucket)

gcloud/storage/test_bucket.py

@@ -37,7 +37,7 @@ def test_ctor_w_implicit_connection(self):
         self.assertEqual(iterator.path, '%s/o' % bucket.path)
         self.assertEqual(iterator.page_number, 0)
         self.assertEqual(iterator.next_page_token, None)
-        self.assertEqual(iterator.prefixes, ())
+        self.assertEqual(iterator.prefixes, set())
 
     def test_ctor_w_explicit_connection(self):
         connection = _Connection()
@@ -48,33 +48,52 @@ def test_ctor_w_explicit_connection(self):
         self.assertEqual(iterator.path, '%s/o' % bucket.path)
         self.assertEqual(iterator.page_number, 0)
         self.assertEqual(iterator.next_page_token, None)
-        self.assertEqual(iterator.prefixes, ())
+        self.assertEqual(iterator.prefixes, set())
 
     def test_get_items_from_response_empty(self):
-        from gcloud.storage._testing import _monkey_defaults
         connection = _Connection()
         bucket = _Bucket()
-        with _monkey_defaults(connection=connection):
-            iterator = self._makeOne(bucket)
-            blobs = list(iterator.get_items_from_response({}))
+        iterator = self._makeOne(bucket, connection=connection)
+        blobs = list(iterator.get_items_from_response({}))
         self.assertEqual(blobs, [])
-        self.assertEqual(iterator.prefixes, ())
+        self.assertEqual(iterator.prefixes, set())
 
     def test_get_items_from_response_non_empty(self):
         from gcloud.storage.blob import Blob
-        from gcloud.storage._testing import _monkey_defaults
         BLOB_NAME = 'blob-name'
         response = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']}
         connection = _Connection()
         bucket = _Bucket()
-        with _monkey_defaults(connection=connection):
-            iterator = self._makeOne(bucket)
-            blobs = list(iterator.get_items_from_response(response))
+        iterator = self._makeOne(bucket, connection=connection)
+        blobs = list(iterator.get_items_from_response(response))
+        self.assertEqual(len(blobs), 1)
+        blob = blobs[0]
+        self.assertTrue(isinstance(blob, Blob))
+        self.assertEqual(blob.name, BLOB_NAME)
+        self.assertEqual(iterator.prefixes, set(['foo']))
+
+    def test_get_items_from_response_cumulative_prefixes(self):
+        from gcloud.storage.blob import Blob
+        BLOB_NAME = 'blob-name1'
+        response1 = {'items': [{'name': BLOB_NAME}], 'prefixes': ['foo']}
+        response2 = {
+            'items': [],
+            'prefixes': ['foo', 'bar'],
+        }
+        connection = _Connection()
+        bucket = _Bucket()
+        iterator = self._makeOne(bucket, connection=connection)
+        # Parse first response.
+        blobs = list(iterator.get_items_from_response(response1))
         self.assertEqual(len(blobs), 1)
         blob = blobs[0]
         self.assertTrue(isinstance(blob, Blob))
         self.assertEqual(blob.name, BLOB_NAME)
-        self.assertEqual(iterator.prefixes, ('foo',))
+        self.assertEqual(iterator.prefixes, set(['foo']))
+        # Parse second response.
+        blobs = list(iterator.get_items_from_response(response2))
+        self.assertEqual(len(blobs), 0)
+        self.assertEqual(iterator.prefixes, set(['foo', 'bar']))
 
 
 class Test_Bucket(unittest2.TestCase):
@@ -243,21 +262,18 @@ def test_get_blob_hit(self):
         self.assertEqual(kw['path'], '/b/%s/o/%s' % (NAME, BLOB_NAME))
 
     def test_list_blobs_defaults(self):
-        from gcloud.storage._testing import _monkey_defaults
         NAME = 'name'
         connection = _Connection({'items': []})
         bucket = self._makeOne(NAME)
-        with _monkey_defaults(connection=connection):
-            iterator = bucket.list_blobs()
-            blobs = list(iterator)
+        iterator = bucket.list_blobs(connection=connection)
+        blobs = list(iterator)
         self.assertEqual(blobs, [])
         kw, = connection._requested
         self.assertEqual(kw['method'], 'GET')
         self.assertEqual(kw['path'], '/b/%s/o' % NAME)
         self.assertEqual(kw['query_params'], {'projection': 'noAcl'})
 
     def test_list_blobs_explicit(self):
-        from gcloud.storage._testing import _monkey_defaults
         NAME = 'name'
         MAX_RESULTS = 10
         PAGE_TOKEN = 'ABCD'
@@ -277,17 +293,17 @@ def test_list_blobs_explicit(self):
         }
         connection = _Connection({'items': []})
         bucket = self._makeOne(NAME)
-        with _monkey_defaults(connection=connection):
-            iterator = bucket.list_blobs(
-                max_results=MAX_RESULTS,
-                page_token=PAGE_TOKEN,
-                prefix=PREFIX,
-                delimiter=DELIMITER,
-                versions=VERSIONS,
-                projection=PROJECTION,
-                fields=FIELDS,
-            )
-            blobs = list(iterator)
+        iterator = bucket.list_blobs(
+            max_results=MAX_RESULTS,
+            page_token=PAGE_TOKEN,
+            prefix=PREFIX,
+            delimiter=DELIMITER,
+            versions=VERSIONS,
+            projection=PROJECTION,
+            fields=FIELDS,
+            connection=connection,
+        )
+        blobs = list(iterator)
         self.assertEqual(blobs, [])
         kw, = connection._requested
         self.assertEqual(kw['method'], 'GET')

regression/storage.py

@@ -259,7 +259,7 @@ def test_root_level_w_delimiter(self):
         self.assertEqual([blob.name for blob in blobs], ['file01.txt'])
         self.assertEqual(iterator.page_number, 1)
         self.assertTrue(iterator.next_page_token is None)
-        self.assertEqual(iterator.prefixes, ('parent/',))
+        self.assertEqual(iterator.prefixes, set(['parent/']))
 
     def test_first_level(self):
         iterator = self.bucket.list_blobs(delimiter='/', prefix='parent/')
@@ -268,7 +268,7 @@ def test_first_level(self):
         self.assertEqual([blob.name for blob in blobs], ['parent/file11.txt'])
         self.assertEqual(iterator.page_number, 1)
         self.assertTrue(iterator.next_page_token is None)
-        self.assertEqual(iterator.prefixes, ('parent/child/',))
+        self.assertEqual(iterator.prefixes, set(['parent/child/']))
 
     def test_second_level(self):
         iterator = self.bucket.list_blobs(delimiter='/',
@@ -281,7 +281,7 @@ def test_second_level(self):
         self.assertEqual(iterator.page_number, 1)
         self.assertTrue(iterator.next_page_token is None)
         self.assertEqual(iterator.prefixes,
-                         ('parent/child/grand/', 'parent/child/other/'))
+                         set(['parent/child/grand/', 'parent/child/other/']))
 
     def test_third_level(self):
         # Pseudo-hierarchy can be arbitrarily deep, subject to the limit
@@ -296,7 +296,7 @@ def test_third_level(self):
                          ['parent/child/grand/file31.txt'])
         self.assertEqual(iterator.page_number, 1)
         self.assertTrue(iterator.next_page_token is None)
-        self.assertEqual(iterator.prefixes, ())
+        self.assertEqual(iterator.prefixes, set())
 
 
 class TestStorageSignURLs(TestStorageFiles):