From ade9985707020dfe95dccad55e22f1cf24916ce7 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Wed, 25 Mar 2020 13:58:09 +0300 Subject: [PATCH 1/7] add conformance tests into client unit tests --- tests/unit/__init__.py | 11 ++++++++++ tests/unit/test__signing.py | 9 +------- tests/unit/test_client.py | 43 +++++++++++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index df379f1e9..a864e9eae 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -11,3 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import io +import json +import os + + +def _read_local_json(json_file): + here = os.path.dirname(__file__) + json_path = os.path.abspath(os.path.join(here, json_file)) + with io.open(json_path, "r", encoding="utf-8-sig") as fileobj: + return json.load(fileobj) diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index 1e1e2776b..d1d2224e3 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -18,9 +18,7 @@ import binascii import calendar import datetime -import io import json -import os import time import unittest @@ -29,12 +27,7 @@ import six from six.moves import urllib_parse - -def _read_local_json(json_file): - here = os.path.dirname(__file__) - json_path = os.path.abspath(os.path.join(here, json_file)) - with io.open(json_path, "r", encoding="utf-8-sig") as fileobj: - return json.load(fileobj) +from . import _read_local_json _SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 8dafd0522..20f07d834 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -14,13 +14,20 @@ import io import json -import unittest - import mock import pytest import requests +import unittest from six.moves import http_client +from google.oauth2.service_account import Credentials +from . import _read_local_json + +_SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") +_CONFORMANCE_TESTS = _read_local_json("url_signer_v4_test_data.json") +_POST_POLICY_TESTS = [test for test in _CONFORMANCE_TESTS if "bucket_name" in test] +_DUMMY_CREDENTIALS = Credentials.from_service_account_info(_SERVICE_ACCOUNT_JSON) + def _make_credentials(): import google.auth.credentials @@ -1742,3 +1749,35 @@ def test_get_signed_policy_v4_with_access_token(self): policy["fields"]["policy"], b"eyJjb25kaXRpb25zIjogW3siYnVja2V0IjogImJ1Y2tldC1uYW1lIn0sIHsiYWNsIjogInByaXZhdGUifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgInRleHQvcGxhaW4iXV0sICJleHBpcmF0aW9uIjogIjIwMjAtMDMtMTJUMDA6MDA6MDAifQ==", ) + + +@pytest.mark.parametrize("test_data", _POST_POLICY_TESTS) +def test_conformance_post_policy(test_data): + import datetime + from google.cloud.storage.client import Client + + client = Client(credentials=_DUMMY_CREDENTIALS) + + bucket_bound_hostname = None + if test_data.get("urlStyle") == "BUCKET_BOUND_HOSTNAME": + bucket_bound_hostname = test_data.get("bucketBoundHostname") + + policy = client.generate_signed_post_policy_v4( + credentials=_DUMMY_CREDENTIALS, + bucket_name=test_data["bucket_name"], + blob_name=test_data["object_name"], + conditions=test_data["conditions"], + fields=test_data.get("fields"), + expiration=datetime.datetime(2020, 3, 25), + virtual_hosted_style=test_data.get("urlStyle") == "VIRTUAL_HOSTED_STYLE", + bucket_bound_hostname=bucket_bound_hostname, + scheme=test_data.get("scheme"), + ) + + assert ( + policy["fields"]["x-goog-credential"] + == "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200325/auto/storage/goog4_request" + ) + assert policy["url"] == test_data["expectedUrl"] + assert policy["fields"]["policy"].decode() == test_data["expectedPolicy"] + assert policy["fields"]["x-goog-signature"] == test_data["expectedSignature"] From f62c48e3d0f25ff5afdb8ece043ff0e635b799d1 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 26 Mar 2020 13:27:31 +0300 Subject: [PATCH 2/7] align conformance tests with test data --- tests/unit/test_client.py | 47 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 20f07d834..3240ac296 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -25,7 +25,7 @@ _SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json") _CONFORMANCE_TESTS = _read_local_json("url_signer_v4_test_data.json") -_POST_POLICY_TESTS = [test for test in _CONFORMANCE_TESTS if "bucket_name" in test] +_POST_POLICY_TESTS = [test for test in _CONFORMANCE_TESTS if "policyInput" in test] _DUMMY_CREDENTIALS = Credentials.from_service_account_info(_SERVICE_ACCOUNT_JSON) @@ -1762,22 +1762,29 @@ def test_conformance_post_policy(test_data): if test_data.get("urlStyle") == "BUCKET_BOUND_HOSTNAME": bucket_bound_hostname = test_data.get("bucketBoundHostname") - policy = client.generate_signed_post_policy_v4( - credentials=_DUMMY_CREDENTIALS, - bucket_name=test_data["bucket_name"], - blob_name=test_data["object_name"], - conditions=test_data["conditions"], - fields=test_data.get("fields"), - expiration=datetime.datetime(2020, 3, 25), - virtual_hosted_style=test_data.get("urlStyle") == "VIRTUAL_HOSTED_STYLE", - bucket_bound_hostname=bucket_bound_hostname, - scheme=test_data.get("scheme"), - ) - - assert ( - policy["fields"]["x-goog-credential"] - == "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200325/auto/storage/goog4_request" - ) - assert policy["url"] == test_data["expectedUrl"] - assert policy["fields"]["policy"].decode() == test_data["expectedPolicy"] - assert policy["fields"]["x-goog-signature"] == test_data["expectedSignature"] + in_data = test_data["policyInput"] + out_data = test_data["policyOutput"] + out_fields = test_data["policyOutput"]["fields"] + + timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + + with mock.patch("google.cloud.storage._signing.NOW", return_value=timestamp): + policy = client.generate_signed_post_policy_v4( + credentials=_DUMMY_CREDENTIALS, + bucket_name=in_data["bucket"], + blob_name=in_data["object"], + conditions=in_data.get("conditions"), + fields=in_data.get("fields"), + expiration=in_data["expiration"], + virtual_hosted_style=in_data.get("urlStyle") == "VIRTUAL_HOSTED_STYLE", + bucket_bound_hostname=bucket_bound_hostname, + scheme=in_data.get("scheme"), + ) + fields = policy["fields"] + + assert policy["url"] == out_data["url"] + assert fields["x-goog-algorithm"] == out_fields["x-goog-algorithm"] + assert fields["x-goog-credential"] == out_fields["x-goog-credential"] + assert fields["x-goog-date"] == out_fields["x-goog-date"] + assert fields["x-goog-signature"] == out_fields["x-goog-signature"] + assert fields["policy"] == out_fields["policy"] From a6309846de1715eb69a1c2a4ce2b7fe4e9ce5022 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 26 Mar 2020 16:33:50 +0300 Subject: [PATCH 3/7] update conformance tests to avoid problems with json spaces and timestamp Z-symbol violation --- tests/unit/test_client.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index eb2a357ce..e84acacd9 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 import io import json import mock @@ -1815,17 +1816,23 @@ def test_conformance_post_policy(test_data): timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") with mock.patch("google.cloud.storage._signing.NOW", return_value=timestamp): - policy = client.generate_signed_post_policy_v4( - credentials=_DUMMY_CREDENTIALS, - bucket_name=in_data["bucket"], - blob_name=in_data["object"], - conditions=in_data.get("conditions"), - fields=in_data.get("fields"), - expiration=in_data["expiration"], - virtual_hosted_style=in_data.get("urlStyle") == "VIRTUAL_HOSTED_STYLE", - bucket_bound_hostname=bucket_bound_hostname, - scheme=in_data.get("scheme"), - ) + with mock.patch( + "google.cloud.storage.client.get_expiration_seconds_v4", + return_value=in_data["expiration"], + ): + with mock.patch("google.cloud.storage.client._NOW", return_value=timestamp): + policy = client.generate_signed_post_policy_v4( + credentials=_DUMMY_CREDENTIALS, + bucket_name=in_data["bucket"], + blob_name=in_data["object"], + conditions=in_data.get("conditions"), + fields=in_data.get("fields"), + expiration=in_data["expiration"], + virtual_hosted_style=in_data.get("urlStyle") + == "VIRTUAL_HOSTED_STYLE", + bucket_bound_hostname=bucket_bound_hostname, + scheme=in_data.get("scheme"), + ) fields = policy["fields"] assert policy["url"] == out_data["url"] @@ -1833,4 +1840,6 @@ def test_conformance_post_policy(test_data): assert fields["x-goog-credential"] == out_fields["x-goog-credential"] assert fields["x-goog-date"] == out_fields["x-goog-date"] assert fields["x-goog-signature"] == out_fields["x-goog-signature"] - assert fields["policy"] == out_fields["policy"] + + decoded_policy = base64.b64decode(fields["policy"]).decode() + assert decoded_policy == out_data["expectedDecodedPolicy"] From c8bce87a322fb027594219fdb2cb3fc78cccea55 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Thu, 26 Mar 2020 17:52:45 +0300 Subject: [PATCH 4/7] fix error with bounded hostnames --- tests/unit/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 85d054684..7fe31702c 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1805,14 +1805,14 @@ def test_conformance_post_policy(test_data): client = Client(credentials=_DUMMY_CREDENTIALS) - bucket_bound_hostname = None - if test_data.get("urlStyle") == "BUCKET_BOUND_HOSTNAME": - bucket_bound_hostname = test_data.get("bucketBoundHostname") - in_data = test_data["policyInput"] out_data = test_data["policyOutput"] out_fields = test_data["policyOutput"]["fields"] + bucket_bound_hostname = None + if in_data.get("urlStyle") == "BUCKET_BOUND_HOSTNAME": + bucket_bound_hostname = in_data["bucketBoundHostname"] + timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") with mock.patch("google.cloud.storage._signing.NOW", return_value=timestamp): From 1c87755f65b23d7ed41de8490a04aa3ede26352d Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Fri, 27 Mar 2020 18:17:33 +0300 Subject: [PATCH 5/7] fix conformance tests --- tests/unit/test_client.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 34c551a54..c2de3c56b 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1797,6 +1797,7 @@ def test_get_signed_policy_v4_with_access_token(self): @pytest.mark.parametrize("test_data", _POST_POLICY_TESTS) def test_conformance_post_policy(test_data): import datetime + import re from google.cloud.storage.client import Client client = Client(credentials=_DUMMY_CREDENTIALS) @@ -1805,12 +1806,24 @@ def test_conformance_post_policy(test_data): out_data = test_data["policyOutput"] out_fields = test_data["policyOutput"]["fields"] - bucket_bound_hostname = None if in_data.get("urlStyle") == "BUCKET_BOUND_HOSTNAME": bucket_bound_hostname = in_data["bucketBoundHostname"] + else: + bucket_bound_hostname = None timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + if "conditions" in in_data: + conditions = [] + for key, value in in_data["conditions"].items(): + field = re.sub(r"(? Date: Fri, 27 Mar 2020 18:26:11 +0300 Subject: [PATCH 6/7] change asserts order --- tests/unit/test_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 0b29e1b51..ca94e9708 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1848,7 +1848,8 @@ def test_conformance_post_policy(test_data): assert fields["x-goog-algorithm"] == out_fields["x-goog-algorithm"] assert fields["x-goog-credential"] == out_fields["x-goog-credential"] assert fields["x-goog-date"] == out_fields["x-goog-date"] - assert fields["x-goog-signature"] == out_fields["x-goog-signature"] - decoded_policy = base64.b64decode(fields["policy"]).decode() + decoded_policy = base64.b64decode(fields["policy"]).decode("utf-8") assert decoded_policy == out_data["expectedDecodedPolicy"] + + assert fields["x-goog-signature"] == out_fields["x-goog-signature"] From 7c4bd6c0b6178371830ce5884795ad963d2125d7 Mon Sep 17 00:00:00 2001 From: IlyaFaer Date: Mon, 30 Mar 2020 16:58:39 +0300 Subject: [PATCH 7/7] fix conformance tests --- tests/unit/test_client.py | 71 +++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index ca94e9708..f108e2afe 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -17,6 +17,7 @@ import json import mock import pytest +import re import requests import unittest from six.moves import http_client @@ -1797,59 +1798,71 @@ def test_get_signed_policy_v4_with_access_token(self): @pytest.mark.parametrize("test_data", _POST_POLICY_TESTS) def test_conformance_post_policy(test_data): import datetime - import re from google.cloud.storage.client import Client - client = Client(credentials=_DUMMY_CREDENTIALS) - in_data = test_data["policyInput"] - out_data = test_data["policyOutput"] - out_fields = test_data["policyOutput"]["fields"] - - if in_data.get("urlStyle") == "BUCKET_BOUND_HOSTNAME": - bucket_bound_hostname = in_data["bucketBoundHostname"] - else: - bucket_bound_hostname = None - timestamp = datetime.datetime.strptime(in_data["timestamp"], "%Y-%m-%dT%H:%M:%SZ") - if "conditions" in in_data: - conditions = [] - for key, value in in_data["conditions"].items(): - field = re.sub(r"(?