diff --git a/storage/google/cloud/storage/_signing.py b/storage/google/cloud/storage/_signing.py index 92703107421d..958b32ec12c5 100644 --- a/storage/google/cloud/storage/_signing.py +++ b/storage/google/cloud/storage/_signing.py @@ -14,7 +14,11 @@ import base64 +import binascii +import collections import datetime +import hashlib +import re import six @@ -23,6 +27,8 @@ NOW = datetime.datetime.utcnow # To be replaced by tests. +MULTIPLE_SPACES_RE = r"\s+" +MULTIPLE_SPACES = re.compile(MULTIPLE_SPACES_RE) def ensure_signed_credentials(credentials): @@ -49,7 +55,7 @@ def ensure_signed_credentials(credentials): ) -def get_signed_query_params(credentials, expiration, string_to_sign): +def get_signed_query_params_v2(credentials, expiration, string_to_sign): """Gets query parameters for creating a signed URL. :type credentials: :class:`google.auth.credentials.Signing` @@ -80,16 +86,16 @@ def get_signed_query_params(credentials, expiration, string_to_sign): } -def get_expiration_seconds(expiration): +def get_expiration_seconds_v2(expiration): """Convert 'expiration' to a number of seconds in the future. - :type expiration: int, long, datetime.datetime, datetime.timedelta - :param expiration: When the signed URL should expire. + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Point in time when the signed URL should expire. :raises: :exc:`TypeError` when expiration is not a valid type. :rtype: int - :returns: a timestamp as an absolute number of seconds. + :returns: a timestamp as an absolute number of seconds since epoch. """ # If it's a timedelta, add it to `now` in UTC. if isinstance(expiration, datetime.timedelta): @@ -109,7 +115,144 @@ def get_expiration_seconds(expiration): return expiration -def generate_signed_url( +_EXPIRATION_TYPES = six.integer_types + (datetime.datetime, datetime.timedelta) + + +def get_expiration_seconds_v4(expiration): + """Convert 'expiration' to a number of seconds offset from the current time. + + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Point in time when the signed URL should expire. + + :raises: :exc:`TypeError` when expiration is not a valid type. + :raises: :exc:`ValueError` when expiration is too large. + :rtype: Integer + :returns: seconds in the future when the signed URL will expire + """ + if not isinstance(expiration, _EXPIRATION_TYPES): + raise TypeError( + "Expected an integer timestamp, datetime, or " + "timedelta. Got %s" % type(expiration) + ) + + now = NOW().replace(tzinfo=_helpers.UTC) + + if isinstance(expiration, six.integer_types): + seconds = expiration + + if isinstance(expiration, datetime.datetime): + + if expiration.tzinfo is None: + expiration = expiration.replace(tzinfo=_helpers.UTC) + + expiration = expiration - now + + if isinstance(expiration, datetime.timedelta): + seconds = int(expiration.total_seconds()) + + if seconds > SEVEN_DAYS: + raise ValueError( + "Max allowed expiration interval is seven days (%d seconds)".format( + SEVEN_DAYS + ) + ) + + return seconds + + +def get_canonical_headers(headers): + """Canonicalize headers for signing. + + See: + https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers + + :type headers: Union[dict|List(Tuple(str,str))] + :param headers: + (Optional) Additional HTTP headers to be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers + Requests using the signed URL *must* pass the specified header + (name and value) with each request for the URL. + + :rtype: str + :returns: List of headers, normalized / sortted per the URL refernced above. + """ + if headers is None: + headers = [] + elif isinstance(headers, dict): + headers = list(headers.items()) + + if not headers: + return [], [] + + normalized = collections.defaultdict(list) + for key, val in headers: + key = key.lower().strip() + val = MULTIPLE_SPACES.sub(" ", val.strip()) + normalized[key].append(val) + + ordered_headers = sorted((key, ",".join(val)) for key, val in normalized.items()) + + canonical_headers = ["{}:{}".format(*item) for item in ordered_headers] + return canonical_headers, ordered_headers + + +_Canonical = collections.namedtuple( + "_Canonical", ["method", "resource", "query_parameters", "headers"] +) + + +def canonicalize(method, resource, query_parameters, headers): + """Canonicalize method, resource + + :type method: str + :param method: The HTTP verb that will be used when requesting the URL. + Defaults to ``'GET'``. If method is ``'RESUMABLE'`` then the + signature will additionally contain the `x-goog-resumable` + header, and the method changed to POST. See the signed URL + docs regarding this flow: + https://cloud.google.com/storage/docs/access-control/signed-urls + + :type resource: str + :param resource: A pointer to a specific resource + (typically, ``/bucket-name/path/to/blob.txt``). + + :type query_parameters: dict + :param query_parameters: + (Optional) Additional query paramtersto be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers#query + + :type headers: Union[dict|List(Tuple(str,str))] + :param headers: + (Optional) Additional HTTP headers to be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers + Requests using the signed URL *must* pass the specified header + (name and value) with each request for the URL. + + :rtype: :class:_Canonical + :returns: Canonical method, resource, query_parameters, and headers. + """ + headers, _ = get_canonical_headers(headers) + + if method == "RESUMABLE": + method = "POST" + headers.append("x-goog-resumable:start") + + if query_parameters is None: + return _Canonical(method, resource, [], headers) + + normalized_qp = sorted( + (key.lower(), value and value.strip() or "") + for key, value in query_parameters.items() + ) + encoded_qp = six.moves.urllib.parse.urlencode(normalized_qp) + canonical_resource = "{}?{}".format(resource, encoded_qp) + return _Canonical(method, canonical_resource, normalized_qp, headers) + + +def generate_signed_url_v2( credentials, resource, expiration, @@ -120,8 +263,10 @@ def generate_signed_url( response_type=None, response_disposition=None, generation=None, + headers=None, + query_parameters=None, ): - """Generate signed URL to provide query-string auth'n to a resource. + """Generate a V2 signed URL to provide query-string auth'n to a resource. .. note:: @@ -151,9 +296,8 @@ def generate_signed_url( :param resource: A pointer to a specific resource (typically, ``/bucket-name/path/to/blob.txt``). - :type expiration: :class:`int`, :class:`long`, :class:`datetime.datetime`, - :class:`datetime.timedelta` - :param expiration: When the signed URL should expire. + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Point in time when the signed URL should expire. :type api_access_endpoint: str :param api_access_endpoint: Optional URI base. Defaults to empty string. @@ -188,6 +332,20 @@ def generate_signed_url( :param generation: (Optional) A value that indicates which generation of the resource to fetch. + :type headers: Union[dict|List(Tuple(str,str))] + :param headers: + (Optional) Additional HTTP headers to be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers + Requests using the signed URL *must* pass the specified header + (name and value) with each request for the URL. + + :type query_parameters: dict + :param query_parameters: + (Optional) Additional query paramtersto be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers#query + :raises: :exc:`TypeError` when expiration is not a valid type. :raises: :exc:`AttributeError` if credentials is not an instance of :class:`google.auth.credentials.Signing`. @@ -196,38 +354,238 @@ def generate_signed_url( :returns: A signed URL you can use to access the resource until expiration. """ - expiration = get_expiration_seconds(expiration) + expiration_stamp = get_expiration_seconds_v2(expiration) - if method == "RESUMABLE": - method = "POST" - canonicalized_resource = "x-goog-resumable:start\n{0}".format(resource) - else: - canonicalized_resource = "{0}".format(resource) + canonical = canonicalize(method, resource, query_parameters, headers) # Generate the string to sign. - string_to_sign = "\n".join( - [ - method, - content_md5 or "", - content_type or "", - str(expiration), - canonicalized_resource, - ] - ) + elements_to_sign = [ + canonical.method, + content_md5 or "", + content_type or "", + str(expiration_stamp), + ] + elements_to_sign.extend(canonical.headers) + elements_to_sign.append(canonical.resource) + string_to_sign = "\n".join(elements_to_sign) # Set the right query parameters. - query_params = get_signed_query_params(credentials, expiration, string_to_sign) + signed_query_params = get_signed_query_params_v2( + credentials, expiration_stamp, string_to_sign + ) if response_type is not None: - query_params["response-content-type"] = response_type + signed_query_params["response-content-type"] = response_type if response_disposition is not None: - query_params["response-content-disposition"] = response_disposition + signed_query_params["response-content-disposition"] = response_disposition if generation is not None: - query_params["generation"] = generation + signed_query_params["generation"] = generation + + signed_query_params.update(canonical.query_parameters) + sorted_signed_query_params = sorted(signed_query_params.items()) # Return the built URL. return "{endpoint}{resource}?{querystring}".format( endpoint=api_access_endpoint, resource=resource, - querystring=six.moves.urllib.parse.urlencode(query_params), + querystring=six.moves.urllib.parse.urlencode(sorted_signed_query_params), + ) + + +SEVEN_DAYS = 7 * 24 * 60 * 60 # max age for V4 signed URLs. +DEFAULT_ENDPOINT = "https://storage.googleapis.com" + + +def generate_signed_url_v4( + credentials, + resource, + expiration, + api_access_endpoint=DEFAULT_ENDPOINT, + method="GET", + content_md5=None, + content_type=None, + response_type=None, + response_disposition=None, + generation=None, + headers=None, + query_parameters=None, + _request_timestamp=None, # for testing only +): + """Generate a V4 signed URL to provide query-string auth'n to a resource. + + .. note:: + + Assumes ``credentials`` implements the + :class:`google.auth.credentials.Signing` interface. Also assumes + ``credentials`` has a ``service_account_email`` property which + identifies the credentials. + + .. note:: + + If you are on Google Compute Engine, you can't generate a signed URL. + Follow `Issue 922`_ for updates on this. If you'd like to be able to + generate a signed URL from GCE, you can use a standard service account + from a JSON file rather than a GCE service account. + + See headers `reference`_ for more details on optional arguments. + + .. _Issue 922: https://github.com/GoogleCloudPlatform/\ + google-cloud-python/issues/922 + .. _reference: https://cloud.google.com/storage/docs/reference-headers + + :type credentials: :class:`google.auth.credentials.Signing` + :param credentials: Credentials object with an associated private key to + sign text. + + :type resource: str + :param resource: A pointer to a specific resource + (typically, ``/bucket-name/path/to/blob.txt``). + + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Point in time when the signed URL should expire. + + :type api_access_endpoint: str + :param api_access_endpoint: Optional URI base. Defaults to + "https://storage.googleapis.com/" + + :type method: str + :param method: The HTTP verb that will be used when requesting the URL. + Defaults to ``'GET'``. If method is ``'RESUMABLE'`` then the + signature will additionally contain the `x-goog-resumable` + header, and the method changed to POST. See the signed URL + docs regarding this flow: + https://cloud.google.com/storage/docs/access-control/signed-urls + + + :type content_md5: str + :param content_md5: (Optional) The MD5 hash of the object referenced by + ``resource``. + + :type content_type: str + :param content_type: (Optional) The content type of the object referenced + by ``resource``. + + :type response_type: str + :param response_type: (Optional) Content type of responses to requests for + the signed URL. Used to over-ride the content type of + the underlying resource. + + :type response_disposition: str + :param response_disposition: (Optional) Content disposition of responses to + requests for the signed URL. + + :type generation: str + :param generation: (Optional) A value that indicates which generation of + the resource to fetch. + + :type headers: dict + :param headers: + (Optional) Additional HTTP headers to be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers + Requests using the signed URL *must* pass the specified header + (name and value) with each request for the URL. + + :type query_parameters: dict + :param query_parameters: + (Optional) Additional query paramtersto be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers#query + + :raises: :exc:`TypeError` when expiration is not a valid type. + :raises: :exc:`AttributeError` if credentials is not an instance + of :class:`google.auth.credentials.Signing`. + + :rtype: str + :returns: A signed URL you can use to access the resource + until expiration. + """ + ensure_signed_credentials(credentials) + expiration_seconds = get_expiration_seconds_v4(expiration) + + if _request_timestamp is None: + now = NOW() + request_timestamp = now.strftime("%Y%m%dT%H%M%SZ") + datestamp = now.date().strftime("%Y%m%d") + else: + request_timestamp = _request_timestamp + datestamp = _request_timestamp[:8] + + client_email = credentials.signer_email + credential_scope = "{}/auto/storage/goog4_request".format(datestamp) + credential = "{}/{}".format(client_email, credential_scope) + + if headers is None: + headers = {} + + if content_type is not None: + headers["Content-Type"] = content_type + + if content_md5 is not None: + headers["Content-MD5"] = content_md5 + + header_names = [key.lower() for key in headers] + if "host" not in header_names: + headers["Host"] = "storage.googleapis.com" + + if method.upper() == "RESUMABLE": + method = "POST" + headers["x-goog-resumable"] = "start" + + canonical_headers, ordered_headers = get_canonical_headers(headers) + canonical_header_string = ( + "\n".join(canonical_headers) + "\n" + ) # Yes, Virginia, the extra newline is part of the spec. + signed_headers = ";".join([key for key, _ in ordered_headers]) + + if query_parameters is None: + query_parameters = {} + else: + query_parameters = {key: value or "" for key, value in query_parameters.items()} + + query_parameters["X-Goog-Algorithm"] = "GOOG4-RSA-SHA256" + query_parameters["X-Goog-Credential"] = credential + query_parameters["X-Goog-Date"] = request_timestamp + query_parameters["X-Goog-Expires"] = expiration_seconds + query_parameters["X-Goog-SignedHeaders"] = signed_headers + + if response_type is not None: + query_parameters["response-content-type"] = response_type + + if response_disposition is not None: + query_parameters["response-content-disposition"] = response_disposition + + if generation is not None: + query_parameters["generation"] = generation + + ordered_query_parameters = sorted(query_parameters.items()) + canonical_query_string = six.moves.urllib.parse.urlencode(ordered_query_parameters) + + canonical_elements = [ + method, + resource, + canonical_query_string, + canonical_header_string, + signed_headers, + "UNSIGNED-PAYLOAD", + ] + canonical_request = "\n".join(canonical_elements) + + canonical_request_hash = hashlib.sha256( + canonical_request.encode("ascii") + ).hexdigest() + + string_elements = [ + "GOOG4-RSA-SHA256", + request_timestamp, + credential_scope, + canonical_request_hash, + ] + string_to_sign = "\n".join(string_elements) + + signature_bytes = credentials.sign_bytes(string_to_sign.encode("ascii")) + signature = binascii.hexlify(signature_bytes).decode("ascii") + + return "{}{}?{}&X-Goog-Signature={}".format( + api_access_endpoint, resource, canonical_query_string, signature ) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index 2fec5c8a1ebc..5126cc16446b 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -54,7 +54,8 @@ from google.api_core.iam import Policy from google.cloud.storage._helpers import _PropertyMixin from google.cloud.storage._helpers import _scalar_property -from google.cloud.storage._signing import generate_signed_url +from google.cloud.storage._signing import generate_signed_url_v2 +from google.cloud.storage._signing import generate_signed_url_v4 from google.cloud.storage.acl import ACL from google.cloud.storage.acl import ObjectACL @@ -302,14 +303,19 @@ def public_url(self): def generate_signed_url( self, - expiration, + expiration=None, + api_access_endpoint=_API_ACCESS_ENDPOINT, method="GET", + content_md5=None, content_type=None, - generation=None, response_disposition=None, response_type=None, + generation=None, + headers=None, + query_parameters=None, client=None, credentials=None, + version=None, ): """Generates a signed URL for this blob. @@ -332,20 +338,23 @@ def generate_signed_url( accessible blobs, but don't want to require users to explicitly log in. - :type expiration: int, long, datetime.datetime, datetime.timedelta - :param expiration: When the signed URL should expire. + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Point in time when the signed URL should expire. + + :type api_access_endpoint: str + :param api_access_endpoint: Optional URI base. :type method: str :param method: The HTTP verb that will be used when requesting the URL. + :type content_md5: str + :param content_md5: (Optional) The MD5 hash of the object referenced by + ``resource``. + :type content_type: str :param content_type: (Optional) The content type of the object referenced by ``resource``. - :type generation: str - :param generation: (Optional) A value that indicates which generation - of the resource to fetch. - :type response_disposition: str :param response_disposition: (Optional) Content disposition of responses to requests for the signed URL. @@ -359,6 +368,24 @@ def generate_signed_url( for the signed URL. Used to over-ride the content type of the underlying blob/object. + :type generation: str + :param generation: (Optional) A value that indicates which generation + of the resource to fetch. + + :type headers: dict + :param headers: + (Optional) Additional HTTP headers to be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers + Requests using the signed URL *must* pass the specified header + (name and value) with each request for the URL. + + :type query_parameters: dict + :param query_parameters: + (Optional) Additional query paramtersto be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers#query + :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: (Optional) The client to use. If not passed, falls back @@ -371,6 +398,11 @@ def generate_signed_url( the URL. Defaults to the credentials stored on the client used. + :type version: str + :param version: (Optional) The version of signed credential to create. + Must be one of 'v2' | 'v4'. + + :raises: :exc:`ValueError` when version is invalid. :raises: :exc:`TypeError` when expiration is not a valid type. :raises: :exc:`AttributeError` if credentials is not an instance of :class:`google.auth.credentials.Signing`. @@ -379,6 +411,11 @@ def generate_signed_url( :returns: A signed URL you can use to access the resource until expiration. """ + if version is None: + version = "v2" + elif version not in ("v2", "v4"): + raise ValueError("'version' must be either 'v2' or 'v4'") + resource = "/{bucket_name}/{quoted_name}".format( bucket_name=self.bucket.name, quoted_name=quote(self.name.encode("utf-8")) ) @@ -387,16 +424,24 @@ def generate_signed_url( client = self._require_client(client) credentials = client._credentials - return generate_signed_url( + if version == "v2": + helper = generate_signed_url_v2 + else: + helper = generate_signed_url_v4 + + return helper( credentials, resource=resource, - api_access_endpoint=_API_ACCESS_ENDPOINT, expiration=expiration, + api_access_endpoint=api_access_endpoint, method=method.upper(), + content_md5=content_md5, content_type=content_type, response_type=response_type, response_disposition=response_disposition, generation=generation, + headers=headers, + query_parameters=query_parameters, ) def exists(self, client=None): diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 2e14aad40d84..0eb6a754ff5e 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -33,6 +33,8 @@ from google.cloud.storage._helpers import _PropertyMixin from google.cloud.storage._helpers import _scalar_property from google.cloud.storage._helpers import _validate_name +from google.cloud.storage._signing import generate_signed_url_v2 +from google.cloud.storage._signing import generate_signed_url_v4 from google.cloud.storage.acl import BucketACL from google.cloud.storage.acl import DefaultObjectACL from google.cloud.storage.blob import Blob @@ -45,6 +47,7 @@ "valid before the bucket is created. Instead, pass the location " "to `Bucket.create`." ) +_API_ACCESS_ENDPOINT = "https://storage.googleapis.com" def _blobs_page_start(iterator, page, response): @@ -1969,3 +1972,109 @@ def lock_retention_policy(self, client=None): method="POST", path=path, query_params=query_params, _target_object=self ) self._set_properties(api_response) + + def generate_signed_url( + self, + expiration=None, + api_access_endpoint=_API_ACCESS_ENDPOINT, + method="GET", + headers=None, + query_parameters=None, + client=None, + credentials=None, + version=None, + ): + """Generates a signed URL for this bucket. + + .. note:: + + If you are on Google Compute Engine, you can't generate a signed + URL using GCE service account. Follow `Issue 50`_ for updates on + this. If you'd like to be able to generate a signed URL from GCE, + you can use a standard service account from a JSON file rather + than a GCE service account. + + .. _Issue 50: https://github.com/GoogleCloudPlatform/\ + google-auth-library-python/issues/50 + + If you have a bucket that you want to allow access to for a set + amount of time, you can use this method to generate a URL that + is only valid within a certain time period. + + This is particularly useful if you don't want publicly + accessible buckets, but don't want to require users to explicitly + log in. + + :type expiration: Union[Integer, datetime.datetime, datetime.timedelta] + :param expiration: Point in time when the signed URL should expire. + + :type api_access_endpoint: str + :param api_access_endpoint: Optional URI base. + + :type method: str + :param method: The HTTP verb that will be used when requesting the URL. + + :type headers: dict + :param headers: + (Optional) Additional HTTP headers to be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers + Requests using the signed URL *must* pass the specified header + (name and value) with each request for the URL. + + :type query_parameters: dict + :param query_parameters: + (Optional) Additional query paramtersto be included as part of the + signed URLs. See: + https://cloud.google.com/storage/docs/xml-api/reference-headers#query + + :type client: :class:`~google.cloud.storage.client.Client` or + ``NoneType`` + :param client: (Optional) The client to use. If not passed, falls back + to the ``client`` stored on the blob's bucket. + + + :type credentials: :class:`oauth2client.client.OAuth2Credentials` or + :class:`NoneType` + :param credentials: (Optional) The OAuth2 credentials to use to sign + the URL. Defaults to the credentials stored on the + client used. + + :type version: str + :param version: (Optional) The version of signed credential to create. + Must be one of 'v2' | 'v4'. + + :raises: :exc:`ValueError` when version is invalid. + :raises: :exc:`TypeError` when expiration is not a valid type. + :raises: :exc:`AttributeError` if credentials is not an instance + of :class:`google.auth.credentials.Signing`. + + :rtype: str + :returns: A signed URL you can use to access the resource + until expiration. + """ + if version is None: + version = "v2" + elif version not in ("v2", "v4"): + raise ValueError("'version' must be either 'v2' or 'v4'") + + resource = "/{bucket_name}".format(bucket_name=self.name) + + if credentials is None: + client = self._require_client(client) + credentials = client._credentials + + if version == "v2": + helper = generate_signed_url_v2 + else: + helper = generate_signed_url_v4 + + return helper( + credentials, + resource=resource, + expiration=expiration, + api_access_endpoint=api_access_endpoint, + method=method.upper(), + headers=headers, + query_parameters=query_parameters, + ) diff --git a/storage/tests/system.py b/storage/tests/system.py index 35d10d408f2a..79a574f3029d 100644 --- a/storage/tests/system.py +++ b/storage/tests/system.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import os -import tempfile import re +import tempfile import time import unittest +import pytest import requests import six @@ -81,6 +83,7 @@ def setUpModule(): def tearDownModule(): + _empty_bucket(Config.TEST_BUCKET) errors = (exceptions.Conflict, exceptions.TooManyRequests) retry = RetryErrors(errors, max_tries=9) retry(_empty_bucket)(Config.TEST_BUCKET) @@ -723,82 +726,205 @@ def test_third_level(self): self.assertEqual(iterator.prefixes, set()) -class TestStorageSignURLs(TestStorageFiles): - def setUp(self): - super(TestStorageSignURLs, self).setUp() +class TestStorageSignURLs(unittest.TestCase): + BLOB_CONTENT = b"This time for sure, Rocky!" + @classmethod + def setUpClass(cls): if ( type(Config.CLIENT._credentials) - is not google.oauth2.service_account.credentials + is not google.oauth2.service_account.Credentials ): - self.skipTest("Signing tests requires a service account credential") - - logo_path = self.FILES["logo"]["path"] - with open(logo_path, "rb") as file_obj: - self.LOCAL_FILE = file_obj.read() + cls.skipTest("Signing tests requires a service account credential") - blob = self.bucket.blob("LogoToSign.jpg") - blob.upload_from_string(self.LOCAL_FILE) - self.case_blobs_to_delete.append(blob) + bucket_name = "gcp-signing" + unique_resource_id() + cls.bucket = Config.CLIENT.create_bucket(bucket_name) + cls.blob = cls.bucket.blob("README.txt") + cls.blob.upload_from_string(cls.BLOB_CONTENT) - def tearDown(self): - errors = (exceptions.TooManyRequests, exceptions.ServiceUnavailable) + @classmethod + def tearDownClass(cls): + _empty_bucket(cls.bucket) + errors = (exceptions.Conflict, exceptions.TooManyRequests) retry = RetryErrors(errors, max_tries=6) - for blob in self.case_blobs_to_delete: - if blob.exists(): - retry(blob.delete)() + retry(cls.bucket.delete)(force=True) - def test_create_signed_read_url(self): - blob = self.bucket.blob("LogoToSign.jpg") - expiration = int(time.time() + 10) - signed_url = blob.generate_signed_url( - expiration, method="GET", client=Config.CLIENT + @staticmethod + def _morph_expiration(version, expiration): + if expiration is not None: + return expiration + + if version == "v2": + return int(time.time()) + 10 + + return 10 + + def _create_signed_list_blobs_url_helper( + self, version, expiration=None, method="GET" + ): + expiration = self._morph_expiration(version, expiration) + + signed_url = self.bucket.generate_signed_url( + expiration=expiration, method=method, client=Config.CLIENT, version=version ) response = requests.get(signed_url) self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, self.LOCAL_FILE) - def test_create_signed_read_url_lowercase_method(self): - blob = self.bucket.blob("LogoToSign.jpg") - expiration = int(time.time() + 10) + def test_create_signed_list_blobs_url_v2(self): + self._create_signed_list_blobs_url_helper(version="v2") + + def test_create_signed_list_blobs_url_v2_w_expiration(self): + now = datetime.datetime.utcnow() + delta = datetime.timedelta(seconds=10) + + self._create_signed_list_blobs_url_helper(expiration=now + delta, version="v2") + + def test_create_signed_list_blobs_url_v4(self): + self._create_signed_list_blobs_url_helper(version="v4") + + def test_create_signed_list_blobs_url_v4_w_expiration(self): + now = datetime.datetime.utcnow() + delta = datetime.timedelta(seconds=10) + self._create_signed_list_blobs_url_helper(expiration=now + delta, version="v4") + + def _create_signed_read_url_helper( + self, + blob_name="LogoToSign.jpg", + method="GET", + version="v2", + payload=None, + expiration=None, + ): + expiration = self._morph_expiration(version, expiration) + + if payload is not None: + blob = self.bucket.blob(blob_name) + blob.upload_from_string(payload) + else: + blob = self.blob + signed_url = blob.generate_signed_url( - expiration, method="get", client=Config.CLIENT + expiration=expiration, method=method, client=Config.CLIENT, version=version ) response = requests.get(signed_url) self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, self.LOCAL_FILE) + if payload is not None: + self.assertEqual(response.content, payload) + else: + self.assertEqual(response.content, self.BLOB_CONTENT) - def test_create_signed_read_url_w_non_ascii_name(self): - blob = self.bucket.blob(u"Caf\xe9.txt") - payload = b"Test signed URL for blob w/ non-ASCII name" - blob.upload_from_string(payload) - self.case_blobs_to_delete.append(blob) + def test_create_signed_read_url_v2(self): + self._create_signed_read_url_helper() - expiration = int(time.time() + 10) - signed_url = blob.generate_signed_url( - expiration, method="GET", client=Config.CLIENT + def test_create_signed_read_url_v4(self): + self._create_signed_read_url_helper(version="v4") + + def test_create_signed_read_url_v2_w_expiration(self): + now = datetime.datetime.utcnow() + delta = datetime.timedelta(seconds=10) + + self._create_signed_read_url_helper(expiration=now + delta) + + def test_create_signed_read_url_v4_w_expiration(self): + now = datetime.datetime.utcnow() + delta = datetime.timedelta(seconds=10) + self._create_signed_read_url_helper(expiration=now + delta, version="v4") + + def test_create_signed_read_url_v2_lowercase_method(self): + self._create_signed_read_url_helper(method="get") + + def test_create_signed_read_url_v4_lowercase_method(self): + self._create_signed_read_url_helper(method="get", version="v4") + + def test_create_signed_read_url_v2_w_non_ascii_name(self): + self._create_signed_read_url_helper( + blob_name=u"Caf\xe9.txt", + payload=b"Test signed URL for blob w/ non-ASCII name", ) - response = requests.get(signed_url) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, payload) + def test_create_signed_read_url_v4_w_non_ascii_name(self): + self._create_signed_read_url_helper( + blob_name=u"Caf\xe9.txt", + payload=b"Test signed URL for blob w/ non-ASCII name", + version="v4", + ) + + def _create_signed_delete_url_helper(self, version="v2", expiration=None): + expiration = self._morph_expiration(version, expiration) + + blob = self.bucket.blob("DELETE_ME.txt") + blob.upload_from_string(b"DELETE ME!") - def test_create_signed_delete_url(self): - blob = self.bucket.blob("LogoToSign.jpg") - expiration = int(time.time() + 283473274) signed_delete_url = blob.generate_signed_url( - expiration, method="DELETE", client=Config.CLIENT + expiration=expiration, + method="DELETE", + client=Config.CLIENT, + version=version, ) response = requests.request("DELETE", signed_delete_url) self.assertEqual(response.status_code, 204) self.assertEqual(response.content, b"") - # Check that the blob has actually been deleted. self.assertFalse(blob.exists()) + def test_create_signed_delete_url_v2(self): + self._create_signed_delete_url_helper() + + def test_create_signed_delete_url_v4(self): + self._create_signed_delete_url_helper(version="v4") + + def _signed_resumable_upload_url_helper(self, version="v2", expiration=None): + expiration = self._morph_expiration(version, expiration) + blob = self.bucket.blob("cruddy.txt") + payload = b"DEADBEEF" + + # Initiate the upload using a signed URL. + signed_resumable_upload_url = blob.generate_signed_url( + expiration=expiration, + method="RESUMABLE", + client=Config.CLIENT, + version=version, + ) + + post_headers = {"x-goog-resumable": "start"} + post_response = requests.post(signed_resumable_upload_url, headers=post_headers) + self.assertEqual(post_response.status_code, 201) + + # Finish uploading the body. + location = post_response.headers["Location"] + put_headers = {"content-length": str(len(payload))} + put_response = requests.put(location, headers=put_headers, data=payload) + self.assertEqual(put_response.status_code, 200) + + # Download using a signed URL and verify. + signed_download_url = blob.generate_signed_url( + expiration=expiration, method="GET", client=Config.CLIENT, version=version + ) + + get_response = requests.get(signed_download_url) + self.assertEqual(get_response.status_code, 200) + self.assertEqual(get_response.content, payload) + + # Finally, delete the blob using a signed URL. + signed_delete_url = blob.generate_signed_url( + expiration=expiration, + method="DELETE", + client=Config.CLIENT, + version=version, + ) + + delete_response = requests.delete(signed_delete_url) + self.assertEqual(delete_response.status_code, 204) + + def test_signed_resumable_upload_url_v2(self): + self._signed_resumable_upload_url_helper(version="v2") + + def test_signed_resumable_upload_url_v4(self): + self._signed_resumable_upload_url_helper(version="v4") + class TestStorageCompose(TestStorageFiles): diff --git a/storage/tests/unit/test__signing.py b/storage/tests/unit/test__signing.py index 01922ca97849..f2609d63e428 100644 --- a/storage/tests/unit/test__signing.py +++ b/storage/tests/unit/test__signing.py @@ -13,53 +13,80 @@ # limitations under the License. import base64 +import binascii import calendar import datetime +import io +import json +import os import time import unittest import mock +import pytest import six from six.moves import urllib_parse -class Test_get_expiration_seconds(unittest.TestCase): +def _read_local_json(json_file): + here = os.path.dirname(__file__) + json_path = os.path.abspath(os.path.join(here, json_file)) + with io.open(json_path, "r", encoding="utf-8-sig") as fileobj: + return json.load(fileobj) + + +_SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") +_CONFORMANCE_TESTS = _read_local_json("url_signer_v4_test_data.json") +_CLIENT_TESTS = [test for test in _CONFORMANCE_TESTS if "bucket" not in test] +_BUCKET_TESTS = [ + test for test in _CONFORMANCE_TESTS if "bucket" in test and not test.get("object") +] +_BLOB_TESTS = [ + test for test in _CONFORMANCE_TESTS if "bucket" in test and test.get("object") +] + + +def _utc_seconds(when): + return int(calendar.timegm(when.timetuple())) + + +class Test_get_expiration_seconds_v2(unittest.TestCase): @staticmethod def _call_fut(expiration): - from google.cloud.storage._signing import get_expiration_seconds + from google.cloud.storage._signing import get_expiration_seconds_v2 - return get_expiration_seconds(expiration) + return get_expiration_seconds_v2(expiration) - @staticmethod - def _utc_seconds(when): - return int(calendar.timegm(when.timetuple())) + def test_w_invalid_expiration_type(self): + with self.assertRaises(TypeError): + self._call_fut(object(), None) - def test_w_invalid(self): - self.assertRaises(TypeError, self._call_fut, object()) - self.assertRaises(TypeError, self._call_fut, None) + def test_w_expiration_none(self): + with self.assertRaises(TypeError): + self._call_fut(None) - def test_w_int(self): + def test_w_expiration_int(self): self.assertEqual(self._call_fut(123), 123) - def test_w_long(self): + def test_w_expiration_long(self): if six.PY3: raise unittest.SkipTest("No long on Python 3") self.assertEqual(self._call_fut(long(123)), 123) # noqa: F821 - def test_w_naive_datetime(self): + def test_w_expiration_naive_datetime(self): expiration_no_tz = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) - utc_seconds = self._utc_seconds(expiration_no_tz) + utc_seconds = _utc_seconds(expiration_no_tz) self.assertEqual(self._call_fut(expiration_no_tz), utc_seconds) - def test_w_utc_datetime(self): + def test_w_expiration_utc_datetime(self): from google.cloud._helpers import UTC expiration_utc = datetime.datetime(2004, 8, 19, 0, 0, 0, 0, UTC) - utc_seconds = self._utc_seconds(expiration_utc) + utc_seconds = _utc_seconds(expiration_utc) self.assertEqual(self._call_fut(expiration_utc), utc_seconds) - def test_w_other_zone_datetime(self): + def test_w_expiration_other_zone_datetime(self): from google.cloud._helpers import _UTC class CET(_UTC): @@ -68,13 +95,13 @@ class CET(_UTC): zone = CET() expiration_other = datetime.datetime(2004, 8, 19, 0, 0, 0, 0, zone) - utc_seconds = self._utc_seconds(expiration_other) + utc_seconds = _utc_seconds(expiration_other) cet_seconds = utc_seconds - (60 * 60) # CET one hour earlier than UTC self.assertEqual(self._call_fut(expiration_other), cet_seconds) - def test_w_timedelta_seconds(self): + def test_w_expiration_timedelta_seconds(self): dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) - utc_seconds = self._utc_seconds(dummy_utcnow) + utc_seconds = _utc_seconds(dummy_utcnow) expiration_as_delta = datetime.timedelta(seconds=10) patch = mock.patch( @@ -86,9 +113,9 @@ def test_w_timedelta_seconds(self): self.assertEqual(result, utc_seconds + 10) utcnow.assert_called_once_with() - def test_w_timedelta_days(self): + def test_w_expiration_timedelta_days(self): dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) - utc_seconds = self._utc_seconds(dummy_utcnow) + utc_seconds = _utc_seconds(dummy_utcnow) expiration_as_delta = datetime.timedelta(days=1) patch = mock.patch( @@ -101,17 +128,128 @@ def test_w_timedelta_days(self): utcnow.assert_called_once_with() -class Test_get_signed_query_params(unittest.TestCase): +class Test_get_expiration_seconds_v4(unittest.TestCase): + @staticmethod + def _call_fut(expiration): + from google.cloud.storage._signing import get_expiration_seconds_v4 + + return get_expiration_seconds_v4(expiration) + + def test_w_invalid_expiration_type(self): + with self.assertRaises(TypeError): + self._call_fut(object(), None) + + def test_w_expiration_none(self): + with self.assertRaises(TypeError): + self._call_fut(None) + + def test_w_expiration_int_gt_seven_days(self): + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + delta = datetime.timedelta(days=10) + expiration_utc = dummy_utcnow + delta + expiration_seconds = _utc_seconds(expiration_utc) + + patch = mock.patch( + "google.cloud.storage._signing.NOW", return_value=dummy_utcnow + ) + + with patch as utcnow: + with self.assertRaises(ValueError): + self._call_fut(expiration_seconds) + utcnow.assert_called_once_with() + + def test_w_expiration_int(self): + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + expiration_seconds = 10 + + patch = mock.patch( + "google.cloud.storage._signing.NOW", return_value=dummy_utcnow + ) + + with patch as utcnow: + result = self._call_fut(expiration_seconds) + + self.assertEqual(result, expiration_seconds) + utcnow.assert_called_once_with() + + def test_w_expiration_naive_datetime(self): + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + delta = datetime.timedelta(seconds=10) + expiration_no_tz = dummy_utcnow + delta + + patch = mock.patch( + "google.cloud.storage._signing.NOW", return_value=dummy_utcnow + ) + with patch as utcnow: + result = self._call_fut(expiration_no_tz) + + self.assertEqual(result, delta.seconds) + utcnow.assert_called_once_with() + + def test_w_expiration_utc_datetime(self): + from google.cloud._helpers import UTC + + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0, UTC) + delta = datetime.timedelta(seconds=10) + expiration_utc = dummy_utcnow + delta + + patch = mock.patch( + "google.cloud.storage._signing.NOW", return_value=dummy_utcnow + ) + with patch as utcnow: + result = self._call_fut(expiration_utc) + + self.assertEqual(result, delta.seconds) + utcnow.assert_called_once_with() + + def test_w_expiration_other_zone_datetime(self): + from google.cloud._helpers import UTC + from google.cloud._helpers import _UTC + + class CET(_UTC): + _tzname = "CET" + _utcoffset = datetime.timedelta(hours=1) + + zone = CET() + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0, UTC) + dummy_cetnow = dummy_utcnow.astimezone(zone) + delta = datetime.timedelta(seconds=10) + expiration_other = dummy_cetnow + delta + + patch = mock.patch( + "google.cloud.storage._signing.NOW", return_value=dummy_utcnow + ) + with patch as utcnow: + result = self._call_fut(expiration_other) + + self.assertEqual(result, delta.seconds) + utcnow.assert_called_once_with() + + def test_w_expiration_timedelta(self): + dummy_utcnow = datetime.datetime(2004, 8, 19, 0, 0, 0, 0) + expiration_as_delta = datetime.timedelta(seconds=10) + + patch = mock.patch( + "google.cloud.storage._signing.NOW", return_value=dummy_utcnow + ) + with patch as utcnow: + result = self._call_fut(expiration_as_delta) + + self.assertEqual(result, expiration_as_delta.total_seconds()) + utcnow.assert_called_once_with() + + +class Test_get_signed_query_params_v2(unittest.TestCase): @staticmethod def _call_fut(credentials, expiration, string_to_sign): - from google.cloud.storage._signing import get_signed_query_params + from google.cloud.storage._signing import get_signed_query_params_v2 - return get_signed_query_params(credentials, expiration, string_to_sign) + return get_signed_query_params_v2(credentials, expiration, string_to_sign) def test_it(self): sig_bytes = b"DEADBEEF" account_name = mock.sentinel.service_account_email - credentials = _make_credentials(signing=True, signer_email=account_name) + credentials = _make_credentials(signer_email=account_name) credentials.sign_bytes.return_value = sig_bytes expiration = 100 string_to_sign = "dummy_signature" @@ -126,91 +264,434 @@ def test_it(self): credentials.sign_bytes.assert_called_once_with(string_to_sign) -class Test_generate_signed_url(unittest.TestCase): +class Test_get_canonical_headers(unittest.TestCase): + @staticmethod + def _call_fut(*args, **kwargs): + from google.cloud.storage._signing import get_canonical_headers + + return get_canonical_headers(*args, **kwargs) + + def test_w_none(self): + headers = None + expected_canonical = [] + expected_ordered = [] + canonical, ordered = self._call_fut(headers) + self.assertEqual(canonical, expected_canonical) + self.assertEqual(ordered, expected_ordered) + + def test_w_dict(self): + headers = {"foo": "Foo 1.2.3", "Bar": " baz,bam,qux "} + expected_canonical = ["bar:baz,bam,qux", "foo:Foo 1.2.3"] + expected_ordered = [tuple(item.split(":")) for item in expected_canonical] + canonical, ordered = self._call_fut(headers) + self.assertEqual(canonical, expected_canonical) + self.assertEqual(ordered, expected_ordered) + + def test_w_list_and_multiples(self): + headers = [ + ("foo", "Foo 1.2.3"), + ("Bar", " baz"), + ("Bar", "bam"), + ("Bar", "qux "), + ] + expected_canonical = ["bar:baz,bam,qux", "foo:Foo 1.2.3"] + expected_ordered = [tuple(item.split(":")) for item in expected_canonical] + canonical, ordered = self._call_fut(headers) + self.assertEqual(canonical, expected_canonical) + self.assertEqual(ordered, expected_ordered) + + def test_w_embedded_ws(self): + headers = {"foo": "Foo\n1.2.3", "Bar": " baz bam qux "} + expected_canonical = ["bar:baz bam qux", "foo:Foo 1.2.3"] + expected_ordered = [tuple(item.split(":")) for item in expected_canonical] + canonical, ordered = self._call_fut(headers) + self.assertEqual(canonical, expected_canonical) + self.assertEqual(ordered, expected_ordered) + + +class Test_canonicalize(unittest.TestCase): @staticmethod def _call_fut(*args, **kwargs): - from google.cloud.storage._signing import generate_signed_url + from google.cloud.storage._signing import canonicalize + + return canonicalize(*args, **kwargs) + + def test_wo_headers_or_query_parameters(self): + method = "GET" + resource = "/bucket/blob" + canonical = self._call_fut(method, resource, None, None) + self.assertEqual(canonical.method, method) + self.assertEqual(canonical.resource, resource) + self.assertEqual(canonical.query_parameters, []) + self.assertEqual(canonical.headers, []) + + def test_w_headers_and_resumable(self): + method = "RESUMABLE" + resource = "/bucket/blob" + headers = [("x-goog-extension", "foobar")] + canonical = self._call_fut(method, resource, None, headers) + self.assertEqual(canonical.method, "POST") + self.assertEqual(canonical.resource, resource) + self.assertEqual(canonical.query_parameters, []) + self.assertEqual( + canonical.headers, ["x-goog-extension:foobar", "x-goog-resumable:start"] + ) + + def test_w_query_paramters(self): + method = "GET" + resource = "/bucket/blob" + query_parameters = {"foo": "bar", "baz": "qux"} + canonical = self._call_fut(method, resource, query_parameters, None) + self.assertEqual(canonical.method, method) + self.assertEqual(canonical.resource, "{}?baz=qux&foo=bar".format(resource)) + self.assertEqual(canonical.query_parameters, [("baz", "qux"), ("foo", "bar")]) + self.assertEqual(canonical.headers, []) - return generate_signed_url(*args, **kwargs) + +class Test_generate_signed_url_v2(unittest.TestCase): + @staticmethod + def _call_fut(*args, **kwargs): + from google.cloud.storage._signing import generate_signed_url_v2 + + return generate_signed_url_v2(*args, **kwargs) def _generate_helper( - self, response_type=None, response_disposition=None, generation=None + self, + api_access_endpoint="", + method="GET", + content_md5=None, + content_type=None, + response_type=None, + response_disposition=None, + generation=None, + headers=None, + query_parameters=None, ): - endpoint = "http://api.example.com" + from six.moves.urllib.parse import urlencode + resource = "/name/path" - credentials = _make_credentials( - signing=True, signer_email="service@example.com" - ) + credentials = _make_credentials(signer_email="service@example.com") credentials.sign_bytes.return_value = b"DEADBEEF" signed = base64.b64encode(credentials.sign_bytes.return_value) signed = signed.decode("ascii") expiration = 1000 + url = self._call_fut( credentials, resource, - expiration, - api_access_endpoint=endpoint, + expiration=expiration, + api_access_endpoint=api_access_endpoint, + method=method, + content_md5=content_md5, + content_type=content_type, response_type=response_type, response_disposition=response_disposition, generation=generation, + headers=headers, + query_parameters=query_parameters, ) # Check the mock was called. - string_to_sign = "\n".join(["GET", "", "", str(expiration), resource]) + method = method.upper() + + if headers is None: + headers = [] + elif isinstance(headers, dict): + headers = sorted(headers.items()) + + elements = [] + expected_resource = resource + if method == "RESUMABLE": + elements.append("POST") + headers.append(("x-goog-resumable", "start")) + else: + elements.append(method) + + if query_parameters is not None: + normalized_qp = { + key.lower(): value and value.strip() or "" + for key, value in query_parameters.items() + } + expected_qp = urlencode(sorted(normalized_qp.items())) + expected_resource = "{}?{}".format(resource, expected_qp) + + elements.append(content_md5 or "") + elements.append(content_type or "") + elements.append(str(expiration)) + elements.extend(["{}:{}".format(*header) for header in headers]) + elements.append(expected_resource) + + string_to_sign = "\n".join(elements) + credentials.sign_bytes.assert_called_once_with(string_to_sign) scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url) - self.assertEqual(scheme, "http") - self.assertEqual(netloc, "api.example.com") + expected_scheme, expected_netloc, _, _, _ = urllib_parse.urlsplit( + api_access_endpoint + ) + self.assertEqual(scheme, expected_scheme) + self.assertEqual(netloc, expected_netloc) self.assertEqual(path, resource) self.assertEqual(frag, "") # Check the URL parameters. - params = urllib_parse.parse_qs(qs) - expected_params = { - "GoogleAccessId": [credentials.signer_email], - "Expires": [str(expiration)], - "Signature": [signed], - } + params = dict(urllib_parse.parse_qsl(qs, keep_blank_values=True)) + + self.assertEqual(params["GoogleAccessId"], credentials.signer_email) + self.assertEqual(params["Expires"], str(expiration)) + self.assertEqual(params["Signature"], signed) + if response_type is not None: - expected_params["response-content-type"] = [response_type] + self.assertEqual(params["response-content-type"], response_type) + if response_disposition is not None: - expected_params["response-content-disposition"] = [response_disposition] + self.assertEqual( + params["response-content-disposition"], response_disposition + ) + if generation is not None: - expected_params["generation"] = [generation] - self.assertEqual(params, expected_params) + self.assertEqual(params["generation"], str(generation)) + + if query_parameters is not None: + for key, value in query_parameters.items(): + value = value.strip() if value else "" + self.assertEqual(params[key].lower(), value) def test_w_expiration_int(self): self._generate_helper() - def test_w_custom_fields(self): + def test_w_endpoint(self): + api_access_endpoint = "https://api.example.com" + self._generate_helper(api_access_endpoint=api_access_endpoint) + + def test_w_method(self): + method = "POST" + self._generate_helper(method=method) + + def test_w_method_resumable(self): + method = "RESUMABLE" + self._generate_helper(method=method) + + def test_w_response_type(self): response_type = "text/plain" + self._generate_helper(response_type=response_type) + + def test_w_response_disposition(self): response_disposition = "attachment; filename=blob.png" + self._generate_helper(response_disposition=response_disposition) + + def test_w_generation(self): generation = "123" - self._generate_helper( - response_type=response_type, - response_disposition=response_disposition, - generation=generation, - ) + self._generate_helper(generation=generation) + + def test_w_custom_headers_dict(self): + self._generate_helper(headers={"x-goog-foo": "bar"}) + + def test_w_custom_headers_list(self): + self._generate_helper(headers=[("x-goog-foo", "bar")]) + + def test_w_custom_query_parameters_w_string_value(self): + self._generate_helper(query_parameters={"bar": "/"}) + + def test_w_custom_query_parameters_w_none_value(self): + self._generate_helper(query_parameters={"qux": None}) def test_with_google_credentials(self): resource = "/name/path" credentials = _make_credentials() expiration = int(time.time() + 5) - self.assertRaises( - AttributeError, - self._call_fut, - credentials, - resource=resource, - expiration=expiration, + with self.assertRaises(AttributeError): + self._call_fut(credentials, resource=resource, expiration=expiration) + + +class Test_generate_signed_url_v4(unittest.TestCase): + DEFAULT_EXPIRATION = 1000 + + @staticmethod + def _call_fut(*args, **kwargs): + from google.cloud.storage._signing import generate_signed_url_v4 + + return generate_signed_url_v4(*args, **kwargs) + + def _generate_helper( + self, + expiration=DEFAULT_EXPIRATION, + api_access_endpoint="", + method="GET", + content_type=None, + content_md5=None, + response_type=None, + response_disposition=None, + generation=None, + headers=None, + query_parameters=None, + ): + now = datetime.datetime(2019, 2, 26, 19, 53, 27) + resource = "/name/path" + signer_email = "service@example.com" + credentials = _make_credentials(signer_email=signer_email) + credentials.sign_bytes.return_value = b"DEADBEEF" + + with mock.patch("google.cloud.storage._signing.NOW", lambda: now): + url = self._call_fut( + credentials, + resource, + expiration=expiration, + api_access_endpoint=api_access_endpoint, + method=method, + content_type=content_type, + content_md5=content_md5, + response_type=response_type, + response_disposition=response_disposition, + generation=generation, + headers=headers, + query_parameters=query_parameters, + ) + + # Check the mock was called. + credentials.sign_bytes.assert_called_once() + + scheme, netloc, path, qs, frag = urllib_parse.urlsplit(url) + + expected_scheme, expected_netloc, _, _, _ = urllib_parse.urlsplit( + api_access_endpoint + ) + self.assertEqual(scheme, expected_scheme) + self.assertEqual(netloc, expected_netloc) + self.assertEqual(path, resource) + self.assertEqual(frag, "") + + # Check the URL parameters. + params = dict(urllib_parse.parse_qsl(qs, keep_blank_values=True)) + self.assertEqual(params["X-Goog-Algorithm"], "GOOG4-RSA-SHA256") + + now_date = now.date().strftime("%Y%m%d") + expected_cred = "{}/{}/auto/storage/goog4_request".format( + signer_email, now_date ) + self.assertEqual(params["X-Goog-Credential"], expected_cred) + + now_stamp = now.strftime("%Y%m%dT%H%M%SZ") + self.assertEqual(params["X-Goog-Date"], now_stamp) + self.assertEqual(params["X-Goog-Expires"], str(self.DEFAULT_EXPIRATION)) + + signed = binascii.hexlify(credentials.sign_bytes.return_value).decode("ascii") + self.assertEqual(params["X-Goog-Signature"], signed) + + if response_type is not None: + self.assertEqual(params["response-content-type"], response_type) + + if response_disposition is not None: + self.assertEqual( + params["response-content-disposition"], response_disposition + ) + + if generation is not None: + self.assertEqual(params["generation"], str(generation)) + + if query_parameters is not None: + for key, value in query_parameters.items(): + value = value.strip() if value else "" + self.assertEqual(params[key].lower(), value) + + def test_w_expiration_too_long(self): + with self.assertRaises(ValueError): + self._generate_helper(expiration=datetime.timedelta(days=8)) + + def test_w_defaults(self): + self._generate_helper() + + def test_w_api_access_endpoint(self): + self._generate_helper(api_access_endpoint="http://api.example.com") + + def test_w_method(self): + self._generate_helper(method="PUT") + + def test_w_method_resumable(self): + self._generate_helper(method="RESUMABLE") + + def test_w_content_type(self): + self._generate_helper(content_type="text/plain") + + def test_w_content_md5(self): + self._generate_helper(content_md5="FACEDACE") + + def test_w_response_type(self): + self._generate_helper(response_type="application/octets") + + def test_w_response_disposition(self): + self._generate_helper(response_disposition="attachment") + + def test_w_generation(self): + self._generate_helper(generation=12345) + + def test_w_custom_host_header(self): + self._generate_helper(headers={"Host": "api.example.com"}) + + def test_w_custom_headers(self): + self._generate_helper(headers={"x-goog-foo": "bar"}) + + def test_w_custom_query_parameters_w_string_value(self): + self._generate_helper(query_parameters={"bar": "/"}) + + def test_w_custom_query_parameters_w_none_value(self): + self._generate_helper(query_parameters={"qux": None}) + + +_DUMMY_SERVICE_ACCOUNT = None + + +def dummy_service_account(): + global _DUMMY_SERVICE_ACCOUNT + + from google.oauth2.service_account import Credentials + + if _DUMMY_SERVICE_ACCOUNT is None: + _DUMMY_SERVICE_ACCOUNT = Credentials.from_service_account_info( + _SERVICE_ACCOUNT_JSON + ) + + return _DUMMY_SERVICE_ACCOUNT + + +def _run_conformance_test(resource, test_data): + credentials = dummy_service_account() + + url = Test_generate_signed_url_v4._call_fut( + credentials, + resource, + expiration=test_data["expiration"], + method=test_data["method"], + _request_timestamp=test_data["timestamp"], + headers=test_data.get("headers"), + ) + + assert url == test_data["expectedUrl"] + + +@pytest.mark.parametrize("test_data", _CLIENT_TESTS) +@pytest.mark.skip(reason="Bucketless URLs not yet supported") +def test_conformance_client(test_data): + pass # pragma: NO COVER + + +@pytest.mark.parametrize("test_data", _BUCKET_TESTS) +def test_conformance_bucket(test_data): + resource = "/{}".format(test_data["bucket"]) + _run_conformance_test(resource, test_data) + + +@pytest.mark.parametrize("test_data", _BLOB_TESTS) +def test_conformance_blob(test_data): + resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) + _run_conformance_test(resource, test_data) -def _make_credentials(signing=False, signer_email=None): +def _make_credentials(signer_email=None): import google.auth.credentials - if signing: + if signer_email: credentials = mock.Mock(spec=google.auth.credentials.Signing) credentials.signer_email = signer_email return credentials diff --git a/storage/tests/unit/test_blob.py b/storage/tests/unit/test_blob.py index db4574cd7675..aeb38b428f05 100644 --- a/storage/tests/unit/test_blob.py +++ b/storage/tests/unit/test_blob.py @@ -351,228 +351,201 @@ def test_public_url_with_non_ascii(self): expected_url = "https://storage.googleapis.com/name/winter%20%E2%98%83" self.assertEqual(blob.public_url, expected_url) - def _basic_generate_signed_url_helper(self, credentials=None): + def test_generate_signed_url_w_invalid_version(self): BLOB_NAME = "blob-name" EXPIRATION = "2014-10-16T20:34:37.000Z" connection = _Connection() client = _Client(connection) bucket = _Bucket(client) blob = self._make_one(BLOB_NAME, bucket=bucket) - URI = ( - "http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - "&Expiration=2014-10-16T20:34:37.000Z" - ) + with self.assertRaises(ValueError): + blob.generate_signed_url(EXPIRATION, version="nonesuch") - SIGNER = _Signer() - with mock.patch("google.cloud.storage.blob.generate_signed_url", new=SIGNER): - signed_uri = blob.generate_signed_url(EXPIRATION, credentials=credentials) - self.assertEqual(signed_uri, URI) + def _generate_signed_url_helper( + self, + version=None, + blob_name="blob-name", + api_access_endpoint=None, + method="GET", + content_md5=None, + content_type=None, + response_type=None, + response_disposition=None, + generation=None, + headers=None, + query_parameters=None, + credentials=None, + expiration=None, + ): + from six.moves.urllib import parse + from google.cloud._helpers import UTC + from google.cloud.storage.blob import _API_ACCESS_ENDPOINT - PATH = "/name/%s" % (BLOB_NAME,) - if credentials is None: - EXPECTED_ARGS = (_Connection.credentials,) - else: - EXPECTED_ARGS = (credentials,) - EXPECTED_KWARGS = { - "api_access_endpoint": "https://storage.googleapis.com", - "expiration": EXPIRATION, - "method": "GET", - "resource": PATH, - "content_type": None, - "response_type": None, - "response_disposition": None, - "generation": None, - } - self.assertEqual(SIGNER._signed, [(EXPECTED_ARGS, EXPECTED_KWARGS)]) + api_access_endpoint = api_access_endpoint or _API_ACCESS_ENDPOINT - def test_generate_signed_url_w_default_method(self): - self._basic_generate_signed_url_helper() + delta = datetime.timedelta(hours=1) + + if expiration is None: + expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + delta - def test_generate_signed_url_w_content_type(self): - BLOB_NAME = "blob-name" - EXPIRATION = "2014-10-16T20:34:37.000Z" connection = _Connection() client = _Client(connection) bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - URI = ( - "http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - "&Expiration=2014-10-16T20:34:37.000Z" - ) - - SIGNER = _Signer() - CONTENT_TYPE = "text/html" - with mock.patch("google.cloud.storage.blob.generate_signed_url", new=SIGNER): - signed_url = blob.generate_signed_url(EXPIRATION, content_type=CONTENT_TYPE) - self.assertEqual(signed_url, URI) - - PATH = "/name/%s" % (BLOB_NAME,) - EXPECTED_ARGS = (_Connection.credentials,) - EXPECTED_KWARGS = { - "api_access_endpoint": "https://storage.googleapis.com", - "expiration": EXPIRATION, - "method": "GET", - "resource": PATH, - "content_type": CONTENT_TYPE, - "response_type": None, - "response_disposition": None, - "generation": None, + blob = self._make_one(blob_name, bucket=bucket) + + if version is None: + effective_version = "v2" + else: + effective_version = version + + to_patch = "google.cloud.storage.blob.generate_signed_url_{}".format( + effective_version + ) + + with mock.patch(to_patch) as signer: + signed_uri = blob.generate_signed_url( + expiration=expiration, + api_access_endpoint=api_access_endpoint, + method=method, + credentials=credentials, + content_md5=content_md5, + content_type=content_type, + response_type=response_type, + response_disposition=response_disposition, + generation=generation, + headers=headers, + query_parameters=query_parameters, + version=version, + ) + + self.assertEqual(signed_uri, signer.return_value) + + if credentials is None: + expected_creds = _Connection.credentials + else: + expected_creds = credentials + + encoded_name = blob_name.encode("utf-8") + expected_resource = "/name/{}".format(parse.quote(encoded_name)) + expected_kwargs = { + "resource": expected_resource, + "expiration": expiration, + "api_access_endpoint": api_access_endpoint, + "method": method.upper(), + "content_md5": content_md5, + "content_type": content_type, + "response_type": response_type, + "response_disposition": response_disposition, + "generation": generation, + "headers": headers, + "query_parameters": query_parameters, } - self.assertEqual(SIGNER._signed, [(EXPECTED_ARGS, EXPECTED_KWARGS)]) + signer.assert_called_once_with(expected_creds, **expected_kwargs) + + def test_generate_signed_url_no_version_passed_warning(self): + self._generate_signed_url_helper() - def test_generate_signed_url_w_credentials(self): + def _generate_signed_url_v2_helper(self, **kw): + version = "v2" + self._generate_signed_url_helper(version, **kw) + + def test_generate_signed_url_v2_w_defaults(self): + self._generate_signed_url_v2_helper() + + def test_generate_signed_url_v2_w_expiration(self): + from google.cloud._helpers import UTC + + expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + self._generate_signed_url_v2_helper(expiration=expiration) + + def test_generate_signed_url_v2_w_non_ascii_name(self): + BLOB_NAME = u"\u0410\u043a\u043a\u043e\u0440\u0434\u044b.txt" + self._generate_signed_url_v2_helper(blob_name=BLOB_NAME) + + def test_generate_signed_url_v2_w_slash_in_name(self): + BLOB_NAME = "parent/child" + self._generate_signed_url_v2_helper(blob_name=BLOB_NAME) + + def test_generate_signed_url_v2_w_endpoint(self): + self._generate_signed_url_v2_helper( + api_access_endpoint="https://api.example.com/v1" + ) + + def test_generate_signed_url_v2_w_method(self): + self._generate_signed_url_v2_helper(method="POST") + + def test_generate_signed_url_v2_w_lowercase_method(self): + self._generate_signed_url_v2_helper(method="get") + + def test_generate_signed_url_v2_w_content_md5(self): + self._generate_signed_url_v2_helper(content_md5="FACEDACE") + + def test_generate_signed_url_v2_w_content_type(self): + self._generate_signed_url_v2_helper(content_type="text.html") + + def test_generate_signed_url_v2_w_response_type(self): + self._generate_signed_url_v2_helper(response_type="text.html") + + def test_generate_signed_url_v2_w_response_disposition(self): + self._generate_signed_url_v2_helper(response_disposition="inline") + + def test_generate_signed_url_v2_w_generation(self): + self._generate_signed_url_v2_helper(generation=12345) + + def test_generate_signed_url_v2_w_headers(self): + self._generate_signed_url_v2_helper(headers={"x-goog-foo": "bar"}) + + def test_generate_signed_url_v2_w_credentials(self): credentials = object() - self._basic_generate_signed_url_helper(credentials=credentials) + self._generate_signed_url_v2_helper(credentials=credentials) - def test_generate_signed_url_lowercase_method(self): - BLOB_NAME = "blob-name" - EXPIRATION = "2014-10-16T20:34:37.000Z" - connection = _Connection() - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - URI = ( - u"http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - u"&Expiration=2014-10-16T20:34:37.000Z" - ) - - SIGNER = _Signer() - with mock.patch("google.cloud.storage.blob.generate_signed_url", new=SIGNER): - signed_url = blob.generate_signed_url(EXPIRATION, method="get") - self.assertEqual(signed_url, URI) - - PATH = "/name/%s" % (BLOB_NAME,) - EXPECTED_ARGS = (_Connection.credentials,) - EXPECTED_KWARGS = { - "api_access_endpoint": "https://storage.googleapis.com", - "expiration": EXPIRATION, - "method": "GET", - "resource": PATH, - "content_type": None, - "response_type": None, - "response_disposition": None, - "generation": None, - } - self.assertEqual(SIGNER._signed, [(EXPECTED_ARGS, EXPECTED_KWARGS)]) + def _generate_signed_url_v4_helper(self, **kw): + version = "v4" + self._generate_signed_url_helper(version, **kw) + + def test_generate_signed_url_v4_w_defaults(self): + self._generate_signed_url_v4_helper() - def test_generate_signed_url_non_ascii(self): + def test_generate_signed_url_v4_w_non_ascii_name(self): BLOB_NAME = u"\u0410\u043a\u043a\u043e\u0440\u0434\u044b.txt" - EXPIRATION = "2014-10-16T20:34:37.000Z" - connection = _Connection() - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - URI = ( - u"http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - u"&Expiration=2014-10-16T20:34:37.000Z" - ) - - SIGNER = _Signer() - with mock.patch("google.cloud.storage.blob.generate_signed_url", new=SIGNER): - signed_url = blob.generate_signed_url(EXPIRATION) - self.assertEqual(signed_url, URI) - - EXPECTED_ARGS = (_Connection.credentials,) - EXPECTED_KWARGS = { - "api_access_endpoint": "https://storage.googleapis.com", - "expiration": EXPIRATION, - "method": "GET", - "resource": "/name/%D0%90%D0%BA%D0%BA%D0%BE%D1%80%D0%B4%D1%8B.txt", - "content_type": None, - "response_type": None, - "response_disposition": None, - "generation": None, - } - self.assertEqual(SIGNER._signed, [(EXPECTED_ARGS, EXPECTED_KWARGS)]) + self._generate_signed_url_v4_helper(blob_name=BLOB_NAME) - def test_generate_signed_url_w_slash_in_name(self): + def test_generate_signed_url_v4_w_slash_in_name(self): BLOB_NAME = "parent/child" - EXPIRATION = "2014-10-16T20:34:37.000Z" - connection = _Connection() - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - URI = ( - "http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - "&Expiration=2014-10-16T20:34:37.000Z" - ) - - SIGNER = _Signer() - with mock.patch("google.cloud.storage.blob.generate_signed_url", new=SIGNER): - signed_url = blob.generate_signed_url(EXPIRATION) - self.assertEqual(signed_url, URI) - - EXPECTED_ARGS = (_Connection.credentials,) - EXPECTED_KWARGS = { - "api_access_endpoint": "https://storage.googleapis.com", - "expiration": EXPIRATION, - "method": "GET", - "resource": "/name/parent/child", - "content_type": None, - "response_type": None, - "response_disposition": None, - "generation": None, - } - self.assertEqual(SIGNER._signed, [(EXPECTED_ARGS, EXPECTED_KWARGS)]) + self._generate_signed_url_v4_helper(blob_name=BLOB_NAME) - def test_generate_signed_url_w_method_arg(self): - BLOB_NAME = "blob-name" - EXPIRATION = "2014-10-16T20:34:37.000Z" - connection = _Connection() - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - URI = ( - "http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - "&Expiration=2014-10-16T20:34:37.000Z" - ) - - SIGNER = _Signer() - with mock.patch("google.cloud.storage.blob.generate_signed_url", new=SIGNER): - signed_uri = blob.generate_signed_url(EXPIRATION, method="POST") - self.assertEqual(signed_uri, URI) - - PATH = "/name/%s" % (BLOB_NAME,) - EXPECTED_ARGS = (_Connection.credentials,) - EXPECTED_KWARGS = { - "api_access_endpoint": "https://storage.googleapis.com", - "expiration": EXPIRATION, - "method": "POST", - "resource": PATH, - "content_type": None, - "response_type": None, - "response_disposition": None, - "generation": None, - } - self.assertEqual(SIGNER._signed, [(EXPECTED_ARGS, EXPECTED_KWARGS)]) - - @mock.patch( - "google.cloud.storage._signing.get_signed_query_params", - return_value={ - "GoogleAccessId": "service-account-name", - "Expires": 12345, - "Signature": "signed-data", - }, - ) - def test_generate_resumable_signed_url(self, mock_get_signed_query_params): - """ - Verify correct behavior of resumable upload URL generation - """ - from google.cloud.storage._signing import get_expiration_seconds - from google.cloud.storage._signing import generate_signed_url - - expiry = get_expiration_seconds(datetime.timedelta(hours=1)) - - signed_url = generate_signed_url( - _make_credentials(), "a-bucket", expiry, method="RESUMABLE" - ) - - self.assertTrue(mock_get_signed_query_params.called) - self.assertGreater(len(signed_url), 0) - self.assertIn("a-bucket", signed_url) - self.assertIn("GoogleAccessId", signed_url) - self.assertIn("Expires", signed_url) - self.assertIn("Signature", signed_url) + def test_generate_signed_url_v4_w_endpoint(self): + self._generate_signed_url_v4_helper( + api_access_endpoint="https://api.example.com/v1" + ) + + def test_generate_signed_url_v4_w_method(self): + self._generate_signed_url_v4_helper(method="POST") + + def test_generate_signed_url_v4_w_lowercase_method(self): + self._generate_signed_url_v4_helper(method="get") + + def test_generate_signed_url_v4_w_content_md5(self): + self._generate_signed_url_v4_helper(content_md5="FACEDACE") + + def test_generate_signed_url_v4_w_content_type(self): + self._generate_signed_url_v4_helper(content_type="text.html") + + def test_generate_signed_url_v4_w_response_type(self): + self._generate_signed_url_v4_helper(response_type="text.html") + + def test_generate_signed_url_v4_w_response_disposition(self): + self._generate_signed_url_v4_helper(response_disposition="inline") + + def test_generate_signed_url_v4_w_generation(self): + self._generate_signed_url_v4_helper(generation=12345) + + def test_generate_signed_url_v4_w_headers(self): + self._generate_signed_url_v4_helper(headers={"x-goog-foo": "bar"}) + + def test_generate_signed_url_v4_w_credentials(self): + credentials = object() + self._generate_signed_url_v4_helper(credentials=credentials) def test_exists_miss(self): NONESUCH = "nonesuch" @@ -3304,18 +3277,6 @@ def delete_blob(self, blob_name, client=None, generation=None): self._deleted.append((blob_name, client, generation)) -class _Signer(object): - def __init__(self): - self._signed = [] - - def __call__(self, *args, **kwargs): - self._signed.append((args, kwargs)) - return ( - "http://example.com/abucket/a-blob-name?Signature=DEADBEEF" - "&Expiration=%s" % kwargs.get("expiration") - ) - - class _Client(object): def __init__(self, connection): self._base_connection = connection diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index 2c70911c005f..31b7d293010e 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -2573,6 +2573,173 @@ def test_lock_retention_policy_w_user_project(self): {"ifMetagenerationMatch": 1234, "userProject": user_project}, ) + def test_generate_signed_url_w_invalid_version(self): + expiration = "2014-10-16T20:34:37.000Z" + connection = _Connection() + client = _Client(connection) + bucket = self._make_one(name="bucket_name", client=client) + with self.assertRaises(ValueError): + bucket.generate_signed_url(expiration, version="nonesuch") + + def _generate_signed_url_helper( + self, + version=None, + bucket_name="bucket-name", + api_access_endpoint=None, + method="GET", + content_md5=None, + content_type=None, + response_type=None, + response_disposition=None, + generation=None, + headers=None, + query_parameters=None, + credentials=None, + expiration=None, + ): + from six.moves.urllib import parse + from google.cloud._helpers import UTC + from google.cloud.storage.blob import _API_ACCESS_ENDPOINT + + api_access_endpoint = api_access_endpoint or _API_ACCESS_ENDPOINT + + delta = datetime.timedelta(hours=1) + + if expiration is None: + expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + delta + + connection = _Connection() + client = _Client(connection) + bucket = self._make_one(name=bucket_name, client=client) + + if version is None: + effective_version = "v2" + else: + effective_version = version + + to_patch = "google.cloud.storage.bucket.generate_signed_url_{}".format( + effective_version + ) + + with mock.patch(to_patch) as signer: + signed_uri = bucket.generate_signed_url( + expiration=expiration, + api_access_endpoint=api_access_endpoint, + method=method, + credentials=credentials, + headers=headers, + query_parameters=query_parameters, + version=version, + ) + + self.assertEqual(signed_uri, signer.return_value) + + if credentials is None: + expected_creds = client._credentials + else: + expected_creds = credentials + + encoded_name = bucket_name.encode("utf-8") + expected_resource = "/{}".format(parse.quote(encoded_name)) + expected_kwargs = { + "resource": expected_resource, + "expiration": expiration, + "api_access_endpoint": api_access_endpoint, + "method": method.upper(), + "headers": headers, + "query_parameters": query_parameters, + } + signer.assert_called_once_with(expected_creds, **expected_kwargs) + + def test_generate_signed_url_no_version_passed_warning(self): + self._generate_signed_url_helper() + + def _generate_signed_url_v2_helper(self, **kw): + version = "v2" + self._generate_signed_url_helper(version, **kw) + + def test_generate_signed_url_v2_w_defaults(self): + self._generate_signed_url_v2_helper() + + def test_generate_signed_url_v2_w_expiration(self): + from google.cloud._helpers import UTC + + expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + self._generate_signed_url_v2_helper(expiration=expiration) + + def test_generate_signed_url_v2_w_endpoint(self): + self._generate_signed_url_v2_helper( + api_access_endpoint="https://api.example.com/v1" + ) + + def test_generate_signed_url_v2_w_method(self): + self._generate_signed_url_v2_helper(method="POST") + + def test_generate_signed_url_v2_w_lowercase_method(self): + self._generate_signed_url_v2_helper(method="get") + + def test_generate_signed_url_v2_w_content_md5(self): + self._generate_signed_url_v2_helper(content_md5="FACEDACE") + + def test_generate_signed_url_v2_w_content_type(self): + self._generate_signed_url_v2_helper(content_type="text.html") + + def test_generate_signed_url_v2_w_response_type(self): + self._generate_signed_url_v2_helper(response_type="text.html") + + def test_generate_signed_url_v2_w_response_disposition(self): + self._generate_signed_url_v2_helper(response_disposition="inline") + + def test_generate_signed_url_v2_w_generation(self): + self._generate_signed_url_v2_helper(generation=12345) + + def test_generate_signed_url_v2_w_headers(self): + self._generate_signed_url_v2_helper(headers={"x-goog-foo": "bar"}) + + def test_generate_signed_url_v2_w_credentials(self): + credentials = object() + self._generate_signed_url_v2_helper(credentials=credentials) + + def _generate_signed_url_v4_helper(self, **kw): + version = "v4" + self._generate_signed_url_helper(version, **kw) + + def test_generate_signed_url_v4_w_defaults(self): + self._generate_signed_url_v2_helper() + + def test_generate_signed_url_v4_w_endpoint(self): + self._generate_signed_url_v4_helper( + api_access_endpoint="https://api.example.com/v1" + ) + + def test_generate_signed_url_v4_w_method(self): + self._generate_signed_url_v4_helper(method="POST") + + def test_generate_signed_url_v4_w_lowercase_method(self): + self._generate_signed_url_v4_helper(method="get") + + def test_generate_signed_url_v4_w_content_md5(self): + self._generate_signed_url_v4_helper(content_md5="FACEDACE") + + def test_generate_signed_url_v4_w_content_type(self): + self._generate_signed_url_v4_helper(content_type="text.html") + + def test_generate_signed_url_v4_w_response_type(self): + self._generate_signed_url_v4_helper(response_type="text.html") + + def test_generate_signed_url_v4_w_response_disposition(self): + self._generate_signed_url_v4_helper(response_disposition="inline") + + def test_generate_signed_url_v4_w_generation(self): + self._generate_signed_url_v4_helper(generation=12345) + + def test_generate_signed_url_v4_w_headers(self): + self._generate_signed_url_v4_helper(headers={"x-goog-foo": "bar"}) + + def test_generate_signed_url_v4_w_credentials(self): + credentials = object() + self._generate_signed_url_v4_helper(credentials=credentials) + class _Connection(object): _delete_bucket = False @@ -2612,6 +2779,13 @@ def api_request(self, **kw): class _Client(object): def __init__(self, connection, project=None): - self._connection = connection self._base_connection = connection self.project = project + + @property + def _connection(self): + return self._base_connection + + @property + def _credentials(self): + return self._base_connection.credentials diff --git a/storage/tests/unit/url_signer_v4_test_account.json b/storage/tests/unit/url_signer_v4_test_account.json new file mode 100644 index 000000000000..5fdc01240ef8 --- /dev/null +++ b/storage/tests/unit/url_signer_v4_test_account.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "dummy-project-id", + "private_key_id": "ffffffffffffffffffffffffffffffffffffffff", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsPzMirIottfQ2\nryjQmPWocSEeGo7f7Q4/tMQXHlXFzo93AGgU2t+clEj9L5loNhLVq+vk+qmnyDz5\nQ04y8jVWyMYzzGNNrGRW/yaYqnqlKZCy1O3bmnNjV7EDbC/jE1ZLBY0U3HaSHfn6\nS9ND8MXdgD0/ulRTWwq6vU8/w6i5tYsU7n2LLlQTl1fQ7/emO9nYcCFJezHZVa0H\nmeWsdHwWsok0skwQYQNIzP3JF9BpR5gJT2gNge6KopDesJeLoLzaX7cUnDn+CAnn\nLuLDwwSsIVKyVxhBFsFXPplgpaQRwmGzwEbf/Xpt9qo26w2UMgn30jsOaKlSeAX8\ncS6ViF+tAgMBAAECggEACKRuJCP8leEOhQziUx8Nmls8wmYqO4WJJLyk5xUMUC22\nSI4CauN1e0V8aQmxnIc0CDkFT7qc9xBmsMoF+yvobbeKrFApvlyzNyM7tEa/exh8\nDGD/IzjbZ8VfWhDcUTwn5QE9DCoon9m1sG+MBNlokB3OVOt8LieAAREdEBG43kJu\nyQTOkY9BGR2AY1FnAl2VZ/jhNDyrme3tp1sW1BJrawzR7Ujo8DzlVcS2geKA9at7\n55ua5GbHz3hfzFgjVXDfnkWzId6aHypUyqHrSn1SqGEbyXTaleKTc6Pgv0PgkJjG\nhZazWWdSuf1T5Xbs0OhAK9qraoAzT6cXXvMEvvPt6QKBgQDXcZKqJAOnGEU4b9+v\nOdoh+nssdrIOBNMu1m8mYbUVYS1aakc1iDGIIWNM3qAwbG+yNEIi2xi80a2RMw2T\n9RyCNB7yqCXXVKLBiwg9FbKMai6Vpk2bWIrzahM9on7AhCax/X2AeOp+UyYhFEy6\nUFG4aHb8THscL7b515ukSuKb5QKBgQDMq+9PuaB0eHsrmL6q4vHNi3MLgijGg/zu\nAXaPygSYAwYW8KglcuLZPvWrL6OG0+CrfmaWTLsyIZO4Uhdj7MLvX6yK7IMnagvk\nL3xjgxSklEHJAwi5wFeJ8ai/1MIuCn8p2re3CbwISKpvf7Sgs/W4196P4vKvTiAz\njcTiSYFIKQKBgCjMpkS4O0TakMlGTmsFnqyOneLmu4NyIHgfPb9cA4n/9DHKLKAT\noaWxBPgatOVWs7RgtyGYsk+XubHkpC6f3X0+15mGhFwJ+CSE6tN+l2iF9zp52vqP\nQwkjzm7+pdhZbmaIpcq9m1K+9lqPWJRz/3XXuqi+5xWIZ7NaxGvRjqaNAoGAdK2b\nutZ2y48XoI3uPFsuP+A8kJX+CtWZrlE1NtmS7tnicdd19AtfmTuUL6fz0FwfW4Su\nlQZfPT/5B339CaEiq/Xd1kDor+J7rvUHM2+5p+1A54gMRGCLRv92FQ4EON0RC1o9\nm2I4SHysdO3XmjmdXmfp4BsgAKJIJzutvtbqlakCgYB+Cb10z37NJJ+WgjDt+yT2\nyUNH17EAYgWXryfRgTyi2POHuJitd64Xzuy6oBVs3wVveYFM6PIKXlj8/DahYX5I\nR2WIzoCNLL3bEZ+nC6Jofpb4kspoAeRporj29SgesK6QBYWHWX2H645RkRGYGpDo\n51gjy9m/hSNqBbH2zmh04A==\n-----END PRIVATE KEY-----\n", + "client_email": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com", + "client_id": "000000000000000000000", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/storage/tests/unit/url_signer_v4_test_data.json b/storage/tests/unit/url_signer_v4_test_data.json new file mode 100644 index 000000000000..807f6cf49a3e --- /dev/null +++ b/storage/tests/unit/url_signer_v4_test_data.json @@ -0,0 +1,122 @@ +[ + { + "description": "Simple GET", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74" + }, + + { + "description": "Simple PUT", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8adff1d4285739e31aa68e73767a46bc5511fde377497dbe08481bf5ceb34e29cc9a59921748d8ec3dd4085b7e9b7772a952afedfcdaecb3ae8352275b8b7c867f204e3db85076220a3127a8a9589302fc1181eae13b9b7fe41109ec8cdc93c1e8bac2d7a0cc32a109ca02d06957211326563ab3d3e678a0ba296e298b5fc5e14593c99d444c94724cc4be97015dbff1dca377b508fa0cb7169195de98d0e4ac96c42b918d28c8d92d33e1bd125ce0fb3cd7ad2c45dae65c22628378f6584971b8bf3945b26f2611eb651e9b6a8648970c1ecf386bb71327b082e7296c4e1ee2fc0bdd8983da80af375c817fb1ad491d0bc22c0f51dba0d66e2cffbc90803e47" + }, + + { + "description": "POST for resumable uploads", + "bucket": "test-bucket", + "object": "test-object", + "method": "POST", + "expiration": 10, + "headers": { + "x-goog-resumable": "start" + }, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable&X-Goog-Signature=4a6d39b23343cedf4c30782aed4b384001828c79ffa3a080a481ea01a640dea0a0ceb58d67a12cef3b243c3f036bb3799c6ee88e8db3eaf7d0bdd4b70a228d0736e07eaa1ee076aff5c6ce09dff1f1f03a0d8ead0d2893408dd3604fdabff553aa6d7af2da67cdba6790006a70240f96717b98f1a6ccb24f00940749599be7ef72aaa5358db63ddd54b2de9e2d6d6a586eac4fe25f36d86fc6ab150418e9c6fa01b732cded226c6d62fc95b72473a4cc55a8257482583fe66d9ab6ede909eb41516a8690946c3e87b0f2052eb0e97e012a14b2f721c42e6e19b8a1cd5658ea36264f10b9b1ada66b8ed5bf7ed7d1708377ac6e5fe608ae361fb594d2e5b24c54" + }, + + { + "description": "Vary expiration and timestamp", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 20, + "timestamp": "20190301T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host&X-Goog-Signature=9669ed5b10664dc594c758296580662912cf4bcc5a4ba0b6bf055bcbf6f34eed7bdad664f534962174a924741a0c273a4f67bc1847cef20192a6beab44223bd9d4fbbd749c407b79997598c30f82ddc269ff47ec09fa3afe74e00616d438df0d96a7d8ad0adacfad1dc3286f864d924fe919fb0dce45d3d975c5afe8e13af2db9cc37ba77835f92f7669b61e94c6d562196c1274529e76cfff1564cc2cad7d5387dc8e12f7a5dfd925685fe92c30b43709eee29fa2f66067472cee5423d1a3a4182fe8cea75c9329d181dc6acad7c393cd04f8bf5bc0515127d8ebd65d80c08e19ad03316053ea60033fd1b1fd85a69c576415da3bf0a3718d9ea6d03e0d66f0" + }, + + { + "description": "Vary bucket and object", + "bucket": "test-bucket2", + "object": "test-object2", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket2/test-object2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=36e3d58dfd3ec1d2dd2f24b5ee372a71e811ffaa2162a2b871d26728d0354270bc116face87127532969c4a3967ed05b7309af741e19c7202f3167aa8c2ac420b61417d6451442bb91d7c822cd17be8783f01e05372769c88913561d27e6660dd8259f0081a71f831be6c50283626cbf04494ac10c394b29bb3bce74ab91548f58a37118a452693cf0483d77561fc9cac8f1765d2c724994cca46a83517a10157ee0347a233a2aaeae6e6ab5e204ff8fc5f54f90a3efdb8301d9fff5475d58cd05b181affd657f48203f4fb133c3a3d355b8eefbd10d5a0a5fd70d06e9515460ad74e22334b2cba4b29cae4f6f285cdb92d8f3126d7a1479ca3bdb69c207d860" + }, + + { + "description": "Simple headers", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "foo": "foo-value", + "BAR": "BAR-value" + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost&X-Goog-Signature=68ecd3b008328ed30d91e2fe37444ed7b9b03f28ed4424555b5161980531ef87db1c3a5bc0265aad5640af30f96014c94fb2dba7479c41bfe1c020eb90c0c6d387d4dd09d4a5df8b60ea50eb6b01cdd786a1e37020f5f95eb8f9b6cd3f65a1f8a8a65c9fcb61ea662959efd9cd73b683f8d8804ef4d6d9b2852419b013368842731359d7f9e6d1139032ceca75d5e67cee5fd0192ea2125e5f2955d38d3d50cf116f3a52e6a62de77f6207f5b95aaa1d7d0f8a46de89ea72e7ea30f21286318d7eba0142232b0deb3a1dc9e1e812a981c66b5ffda3c6b01a8a9d113155792309fd53a3acfd054ca7776e8eec28c26480cd1e3c812f67f91d14217f39a606669d" + }, + + { + "description": "Headers should be trimmed", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "leading": " xyz", + "trailing": "abc ", + "collapsed": "abc def" + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btrailing&X-Goog-Signature=1839511d6238d9ac2bbcbba8b23515b3757db35dfa7b8f9bc4b8b4aa270224df747c812526f1a3bcf294d67ed84cd14e074c36bc090e0a542782934a7c925af4a5ea68123e97533704ce8b08ccdf5fe6b412f89c9fc4de243e29abdb098382c5672188ee3f6fef7131413e252c78e7a35658825ad842a50609e9cc463731e17284ff7a14824c989f87cef22fb99dfec20cfeed69d8b3a08f00b43b8284eecd535e50e982b05cd74c5750cd5f986cfc21a2a05f7f3ab7fc31bd684ed1b823b64d29281e923fc6580c49005552ca19c253de087d9d2df881144e44eda40965cfdb4889bf3a35553c9809f4ed20b8355be481b92b9618952b6a04f3017b36053e15" + }, + + { + "description": "Header value with multiple inline values", + "bucket": "test-bucket", + "object": "test-object", + "headers": { + "multiple": " xyz , abc, def , xyz " + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bmultiple&X-Goog-Signature=5cc113735625341f59c7203f0c2c9febc95ba6af6b9c38814f8e523214712087dc0996e4960d273ae1889f248ac1e58d4d19cb3a69ad7670e9a8ca1b434e878f59339dc7006cf32dfd715337e9f593e0504371839174962a08294586e0c78160a7aa303397888c8350637c6af3b32ac310886cc4590bfda9ca561ee58fb5b8ec56bc606d2ada6e7df31f4276e9dcb96bcaea39dc2cd096f3fad774f9c4b30e317ad43736c05f76831437f44e8726c1e90d3f6c9827dc273f211f32fc85658dfc5d357eb606743a6b00a29e519eef1bebaf9db3e8f4b1f5f9afb648ad06e60bc42fa8b57025056697c874c9ea76f5a73201c9717ea43e54713ff3502ff3fc626b" + }, + + { + "description": "Customer-supplied encryption key", + "bucket": "test-bucket", + "object": "test-object", + "headers": + { + "X-Goog-Encryption-Key": "key", + "X-Goog-Encryption-Key-Sha256": "key-hash", + "X-Goog-Encryption-Algorithm": "AES256" + }, + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-encryption-algorithm%3Bx-goog-encryption-key%3Bx-goog-encryption-key-sha256&X-Goog-Signature=278a1c5a3bad248637054a047014760353942433955871031ed08f515b54588654ad033e91f046ab202b68673030e117d1b786c325e870238b035ba75b3feed560a17aff9bab6bddebd4a31a52cb68b214e27d3b0bd886502c6b36b164306fe88b5a07c6063592afe746b2a5d205dbe90dd5386b94f0a78f75d9f53ee884e18f476e8fc2eb1dd910ce0b4ae1f5d7b09876ef9bf983f539c028429e14bad3c75dbd4ed1ae37856f6d6f8a1805eaf8b52a0d6fc993902e4c1ee8de477661f7b67c3663000474cb00e178189789b2a3ed6bd21b4ade684fca8108ac4dd106acb17f5954d045775f7aa5a98ebda5d3075e11a8ea49c64c6ad1481e463e8c9f11f704" + }, + + { + "description": "List Objects", + "bucket": "test-bucket", + "object": "", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b" + } +]