From c3772195292e14f6aeb688ddd18b21fc6d5a541e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 3 Dec 2020 14:15:21 -0600 Subject: [PATCH 1/3] feat: convert `BIGNUMERIC` values to decimal objects Towards #367 --- google/cloud/bigquery/_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 35129d844..502780eda 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -188,6 +188,7 @@ def _record_from_json(value, field): "FLOAT": _float_from_json, "FLOAT64": _float_from_json, "NUMERIC": _decimal_from_json, + "BIGNUMERIC": _decimal_from_json, "BOOLEAN": _bool_from_json, "BOOL": _bool_from_json, "STRING": _string_from_json, From a30ddaf262bcfea16f54dd7f54864ad498614312 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 4 Dec 2020 15:36:48 -0600 Subject: [PATCH 2/3] test: add unit tests for BIGNUMERIC --- google/cloud/bigquery/_helpers.py | 1 + tests/unit/test_client.py | 99 ++++++++++++++++++++++++------- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 502780eda..73809ef99 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -348,6 +348,7 @@ def _time_to_json(value): "FLOAT": _float_to_json, "FLOAT64": _float_to_json, "NUMERIC": _decimal_to_json, + "BIGNUMERIC": _decimal_to_json, "BOOLEAN": _bool_to_json, "BOOL": _bool_to_json, "BYTES": _bytes_to_json, diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index c4bdea2f8..2ecfe89de 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -6290,38 +6290,43 @@ def test_insert_rows_w_numeric(self): creds = _make_credentials() http = object() client = self._make_one(project=project, credentials=creds, _http=http) - conn = client._connection = make_connection({}) table_ref = DatasetReference(project, ds_id).table(table_id) - schema = [SchemaField("account", "STRING"), SchemaField("balance", "NUMERIC")] - insert_table = table.Table(table_ref, schema=schema) rows = [ ("Savings", decimal.Decimal("23.47")), ("Checking", decimal.Decimal("1.98")), ("Mortgage", decimal.Decimal("-12345678909.87654321")), ] + schemas = [ + [SchemaField("account", "STRING"), SchemaField("balance", "NUMERIC")], + [SchemaField("account", "STRING"), SchemaField("balance", "BIGNUMERIC")], + ] - with mock.patch("uuid.uuid4", side_effect=map(str, range(len(rows)))): - errors = client.insert_rows(insert_table, rows) + for schema in schemas: + conn = client._connection = make_connection({}) - self.assertEqual(len(errors), 0) - rows_json = [ - {"account": "Savings", "balance": "23.47"}, - {"account": "Checking", "balance": "1.98"}, - {"account": "Mortgage", "balance": "-12345678909.87654321"}, - ] - sent = { - "rows": [ - {"json": row, "insertId": str(i)} for i, row in enumerate(rows_json) + insert_table = table.Table(table_ref, schema=schema) + with mock.patch("uuid.uuid4", side_effect=map(str, range(len(rows)))): + errors = client.insert_rows(insert_table, rows) + + self.assertEqual(len(errors), 0) + rows_json = [ + {"account": "Savings", "balance": "23.47"}, + {"account": "Checking", "balance": "1.98"}, + {"account": "Mortgage", "balance": "-12345678909.87654321"}, ] - } - conn.api_request.assert_called_once_with( - method="POST", - path="/projects/{}/datasets/{}/tables/{}/insertAll".format( - project, ds_id, table_id - ), - data=sent, - timeout=None, - ) + sent = { + "rows": [ + {"json": row, "insertId": str(i)} for i, row in enumerate(rows_json) + ] + } + conn.api_request.assert_called_once_with( + method="POST", + path="/projects/{}/datasets/{}/tables/{}/insertAll".format( + project, ds_id, table_id + ), + data=sent, + timeout=None, + ) @unittest.skipIf(pandas is None, "Requires `pandas`") def test_insert_rows_from_dataframe(self): @@ -6922,6 +6927,54 @@ def test_list_rows_query_params(self): req = conn.api_request.call_args_list[i] self.assertEqual(req[1]["query_params"], test[1], "for kwargs %s" % test[0]) + def test_list_rows_w_numeric(self): + from google.cloud.bigquery.schema import SchemaField + from google.cloud.bigquery.table import Table + + resource = { + "totalRows": 3, + "rows": [ + { + "f": [ + {"v": "-1.23456789"}, + {"v": "-123456789.987654321"}, + ] + }, + { + "f": [ + {"v": None}, + {"v": "3.141592653589793238462643383279502884"}, + ] + }, + { + "f": [ + {"v": "2718281828459045235360287471.352662497"}, + {"v": None}, + ] + }, + ], + } + creds = _make_credentials() + http = object() + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) + conn = client._connection = make_connection(resource) + schema = [ + SchemaField("num", "NUMERIC"), + SchemaField("bignum", "BIGNUMERIC"), + ] + table = Table(self.TABLE_REF, schema=schema) + + iterator = client.list_rows(table) + rows = list(iterator) + + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0]["num"], decimal.Decimal("-1.23456789")) + self.assertEqual(rows[0]["bignum"], decimal.Decimal("-123456789.987654321")) + self.assertIsNone(rows[1]["num"]) + self.assertEqual(rows[1]["bignum"], decimal.Decimal("3.141592653589793238462643383279502884")) + self.assertEqual(rows[2]["num"], decimal.Decimal("2718281828459045235360287471.352662497")) + self.assertIsNone(rows[2]["bignum"]) + def test_list_rows_repeated_fields(self): from google.cloud.bigquery.schema import SchemaField From 9da3953a854170d363d9535bcd4368eb03ddf1ed Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 4 Dec 2020 15:50:42 -0600 Subject: [PATCH 3/3] blacken --- tests/unit/test_client.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 2ecfe89de..ea4924d38 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -6934,30 +6934,15 @@ def test_list_rows_w_numeric(self): resource = { "totalRows": 3, "rows": [ - { - "f": [ - {"v": "-1.23456789"}, - {"v": "-123456789.987654321"}, - ] - }, - { - "f": [ - {"v": None}, - {"v": "3.141592653589793238462643383279502884"}, - ] - }, - { - "f": [ - {"v": "2718281828459045235360287471.352662497"}, - {"v": None}, - ] - }, + {"f": [{"v": "-1.23456789"}, {"v": "-123456789.987654321"}]}, + {"f": [{"v": None}, {"v": "3.141592653589793238462643383279502884"}]}, + {"f": [{"v": "2718281828459045235360287471.352662497"}, {"v": None}]}, ], } creds = _make_credentials() http = object() client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) - conn = client._connection = make_connection(resource) + client._connection = make_connection(resource) schema = [ SchemaField("num", "NUMERIC"), SchemaField("bignum", "BIGNUMERIC"), @@ -6971,8 +6956,12 @@ def test_list_rows_w_numeric(self): self.assertEqual(rows[0]["num"], decimal.Decimal("-1.23456789")) self.assertEqual(rows[0]["bignum"], decimal.Decimal("-123456789.987654321")) self.assertIsNone(rows[1]["num"]) - self.assertEqual(rows[1]["bignum"], decimal.Decimal("3.141592653589793238462643383279502884")) - self.assertEqual(rows[2]["num"], decimal.Decimal("2718281828459045235360287471.352662497")) + self.assertEqual( + rows[1]["bignum"], decimal.Decimal("3.141592653589793238462643383279502884") + ) + self.assertEqual( + rows[2]["num"], decimal.Decimal("2718281828459045235360287471.352662497") + ) self.assertIsNone(rows[2]["bignum"]) def test_list_rows_repeated_fields(self):