Skip to content

Firestore merge option #4847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
chemelnucfin wants to merge 2 commits into from
Closed

Firestore merge option #4847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d7b462
Firestore: Remove CreateIfMissing option
chemelnucfin Dec 22, 2017
b450adf
Firestore: MergeOption
chemelnucfin Feb 7, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions firestore/google/cloud/firestore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from google.cloud.firestore_v1beta1 import AdminClient
from google.cloud.firestore_v1beta1 import Client
from google.cloud.firestore_v1beta1 import CollectionReference
from google.cloud.firestore_v1beta1 import CreateIfMissingOption
from google.cloud.firestore_v1beta1 import DELETE_FIELD
from google.cloud.firestore_v1beta1 import DocumentReference
from google.cloud.firestore_v1beta1 import DocumentSnapshot
Expand All @@ -42,7 +41,6 @@
'AdminClient',
'Client',
'CollectionReference',
'CreateIfMissingOption',
'DELETE_FIELD',
'DocumentReference',
'DocumentSnapshot',
Expand Down
2 changes: 0 additions & 2 deletions firestore/google/cloud/firestore_v1beta1/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from google.cloud.firestore_v1beta1._helpers import ReadAfterWriteError
from google.cloud.firestore_v1beta1.batch import WriteBatch
from google.cloud.firestore_v1beta1.client import Client
from google.cloud.firestore_v1beta1.client import CreateIfMissingOption
from google.cloud.firestore_v1beta1.client import ExistsOption
from google.cloud.firestore_v1beta1.client import LastUpdateOption
from google.cloud.firestore_v1beta1.client import WriteOption
Expand All @@ -46,7 +45,6 @@
'AdminClient',
'Client',
'CollectionReference',
'CreateIfMissingOption',
'DELETE_FIELD',
'DocumentReference',
'DocumentSnapshot',
Expand Down
16 changes: 5 additions & 11 deletions firestore/google/cloud/firestore_v1beta1/_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,9 @@
import grpc
import six

from google.cloud import exceptions
from google.cloud._helpers import _datetime_to_pb_timestamp
from google.cloud._helpers import _pb_timestamp_to_datetime
from google.cloud import exceptions

from google.cloud.firestore_v1beta1 import constants
from google.cloud.firestore_v1beta1.gapic import enums
from google.cloud.firestore_v1beta1.proto import common_pb2
Expand All @@ -47,10 +46,6 @@
'The data at {!r} is not a dictionary, so it cannot contain the key {!r}')
FIELD_PATH_DELIMITER = '.'
DOCUMENT_PATH_DELIMITER = '/'
_NO_CREATE_TEMPLATE = (
'The ``create_if_missing`` option cannot be used '
'on ``{}()`` requests.')
NO_CREATE_ON_DELETE = _NO_CREATE_TEMPLATE.format('delete')
INACTIVE_TXN = (
'Transaction not in progress, cannot be used in API requests.')
READ_AFTER_WRITE_ERROR = 'Attempted read after write in a transaction.'
Expand Down Expand Up @@ -817,15 +812,14 @@ def pbs_for_set(document_path, document_data, option):
or two ``Write`` protobuf instances for ``set()``.
"""
transform_paths, actual_data = remove_server_timestamp(document_data)

update_pb = write_pb2.Write(
update=document_pb2.Document(
name=document_path,
fields=encode_dict(actual_data),
),
)
if option is not None:
option.modify_write(update_pb)
option.modify_write(update_pb, actual_data=actual_data, path=document_path)

write_pbs = [update_pb]
if transform_paths:
Expand Down Expand Up @@ -856,7 +850,7 @@ def pbs_for_update(client, document_path, field_updates, option):
"""
if option is None:
# Default uses ``exists=True``.
option = client.write_option(create_if_missing=False)
option = client.write_option(exists=True)

transform_paths, actual_updates = remove_server_timestamp(field_updates)
update_values, field_paths = FieldPathHelper.to_field_paths(actual_updates)
Expand All @@ -870,7 +864,7 @@ def pbs_for_update(client, document_path, field_updates, option):
update_mask=common_pb2.DocumentMask(field_paths=sorted(field_paths)),
)
# Due to the default, we don't have to check if ``None``.
option.modify_write(update_pb)
option.modify_write(update_pb, field_paths=field_paths)
write_pbs = [update_pb]

if transform_paths:
Expand All @@ -897,7 +891,7 @@ def pb_for_delete(document_path, option):
"""
write_pb = write_pb2.Write(delete=document_path)
if option is not None:
option.modify_write(write_pb, no_create_msg=NO_CREATE_ON_DELETE)
option.modify_write(write_pb)

return write_pb

Expand Down
89 changes: 30 additions & 59 deletions firestore/google/cloud/firestore_v1beta1/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,15 @@
from google.cloud.firestore_v1beta1.document import DocumentSnapshot
from google.cloud.firestore_v1beta1.gapic import firestore_client
from google.cloud.firestore_v1beta1.transaction import Transaction
from google.cloud.firestore_v1beta1.proto import common_pb2
from google.cloud.firestore_v1beta1.proto import document_pb2
from google.cloud.firestore_v1beta1.proto import write_pb2


DEFAULT_DATABASE = '(default)'
"""str: The default database used in a :class:`~.firestore.client.Client`."""
_BAD_OPTION_ERR = (
'Exactly one of ``create_if_missing``, ``last_update_time`` '
'and ``exists`` must be provided.')
'Exactly one of ``last_update_time`` or ``exists`` must be provided.')
_BAD_DOC_TEMPLATE = (
'Document {!r} appeared in response but was not present among references')
_ACTIVE_TXN = 'There is already an active transaction.'
Expand Down Expand Up @@ -252,10 +254,8 @@ def write_option(**kwargs):
:meth:`~.DocumentReference.update` and
:meth:`~.DocumentReference.delete`.

Exactly one of three keyword arguments must be provided:
One of the following two keyword arguments must be provided:

* ``create_if_missing`` (:class:`bool`): Indicates if the document
should be created if it doesn't already exist.
* ``last_update_time`` (:class:`google.protobuf.timestamp_pb2.\
Timestamp`): A timestamp. When set, the target document must exist
and have been last updated at that time. Protobuf ``update_time``
Expand All @@ -268,9 +268,8 @@ def write_option(**kwargs):
it is not allowed). Providing multiple would be an apparent
contradiction, since ``last_update_time`` assumes that the
document **was** updated (it can't have been updated if it
doesn't exist) and both ``create_if_missing`` and ``exists`` indicate
that it is unknown if the document exists or not (but in different
ways).
doesn't exist) and ``exists`` indicate that it is unknown if the
document exists or not.

Args:
kwargs (Dict[str, Any]): The keyword arguments described above.
Expand All @@ -283,12 +282,12 @@ def write_option(**kwargs):
raise TypeError(_BAD_OPTION_ERR)

name, value = kwargs.popitem()
if name == 'create_if_missing':
return CreateIfMissingOption(value)
elif name == 'last_update_time':
if name == 'last_update_time':
return LastUpdateOption(value)
elif name == 'exists':
return ExistsOption(value)
elif name == 'merge':
return MergeOption()
else:
extra = '{!r} was provided'.format(name)
raise TypeError(_BAD_OPTION_ERR, extra)
Expand Down Expand Up @@ -424,47 +423,37 @@ def modify_write(self, write_pb, **unused_kwargs):
write_pb.current_document.CopyFrom(current_doc)


class CreateIfMissingOption(WriteOption):
"""Option used to assert "create if missing" on a write operation.
class MergeOption(WriteOption):
"""Option used to merge on a write operation.

This will typically be created by
:meth:`~.firestore_v1beta1.client.Client.write_option`.

Args:
create_if_missing (bool): Indicates if the document should be created
if it doesn't already exist.
"""

def __init__(self, create_if_missing):
self._create_if_missing = create_if_missing

def modify_write(self, write_pb, no_create_msg=None):
def modify_write(self, write_pb, actual_data=None, path=None, **unused_kwargs):
"""Modify a ``Write`` protobuf based on the state of this write option.

If:

* ``create_if_missing=False``, adds a precondition that requires
existence
* ``create_if_missing=True``, does not add any precondition
* ``no_create_msg`` is passed, raises an exception. For example, in a
:meth:`~.DocumentReference.delete`, no "create" can occur, so it
wouldn't make sense to "create if missing".

Args:
write_pb (google.cloud.firestore_v1beta1.types.Write): A
``Write`` protobuf instance to be modified with a precondition
determined by the state of this option.
no_create_msg (Optional[str]): A message to use to indicate that
a create operation is not allowed.

Raises:
ValueError: If ``no_create_msg`` is passed.
actual_data (dict):
The actual field names and values to use for replacing a
document.
path (str): A fully-qualified document_path
unused_kwargs (Dict[str, Any]): Keyword arguments accepted by
other subclasses that are unused here.
"""
if no_create_msg is not None:
raise ValueError(no_create_msg)
elif not self._create_if_missing:
current_doc = types.Precondition(exists=True)
write_pb.current_document.CopyFrom(current_doc)
actual_data, field_paths = _helpers.FieldPathHelper.to_field_paths(actual_data)
doc = document_pb2.Document(
name=path,
fields=_helpers.encode_dict(actual_data)
)
write = write_pb2.Write(
update=doc,
)
write_pb.CopyFrom(write)
mask = common_pb2.DocumentMask(field_paths=sorted(field_paths))
write_pb.update_mask.CopyFrom(mask)


class ExistsOption(WriteOption):
Expand All @@ -473,24 +462,6 @@ class ExistsOption(WriteOption):
This will typically be created by
:meth:`~.firestore_v1beta1.client.Client.write_option`.

This option is closely related to
:meth:`~.firestore_v1beta1.client.CreateIfMissingOption`,
but a "create if missing". In fact,

.. code-block:: python

>>> ExistsOption(exists=True)

is (mostly) equivalent to

.. code-block:: python

>>> CreateIfMissingOption(create_if_missing=False)

The only difference being that "create if missing" cannot be used
on some operations (e.g. :meth:`~.DocumentReference.delete`)
while "exists" can.

Args:
exists (bool): Indicates if the document being modified
should already exist.
Expand Down
7 changes: 1 addition & 6 deletions firestore/google/cloud/firestore_v1beta1/document.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,19 +377,14 @@ def delete(self, option=None):
Args:
option (Optional[~.firestore_v1beta1.client.WriteOption]): A
write option to make assertions / preconditions on the server
state of the document before applying changes. Note that
``create_if_missing`` can't be used here since it does not
apply (i.e. a "delete" cannot "create").
state of the document before applying changes.

Returns:
google.protobuf.timestamp_pb2.Timestamp: The time that the delete
request was received by the server. If the document did not exist
when the delete was sent (i.e. nothing was deleted), this method
will still succeed and will still return the time that the
request was received by the server.

Raises:
ValueError: If the ``create_if_missing`` write option is used.
"""
write_pb = _helpers.pb_for_delete(self._document_path, option)
with _helpers.remap_gax_error_on_commit():
Expand Down
49 changes: 45 additions & 4 deletions firestore/tests/system.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def test_document_set(client, cleanup):
cleanup(document)

# 0. Make sure the document doesn't exist yet using an option.
option0 = client.write_option(create_if_missing=False)
option0 = client.write_option(exists=True)
with pytest.raises(NotFound) as exc_info:
document.set({'no': 'way'}, option=option0)

Expand All @@ -148,7 +148,7 @@ def test_document_set(client, cleanup):

# 1. Use ``set()`` to create the document (using an option).
data1 = {'foo': 88}
option1 = client.write_option(create_if_missing=True)
option1 = client.write_option(exists=False)
write_result1 = document.set(data1, option=option1)
snapshot1 = document.get()
assert snapshot1.to_dict() == data1
Expand Down Expand Up @@ -194,6 +194,47 @@ def test_document_set(client, cleanup):
assert exc_to_code(exc_info.value.cause) == StatusCode.FAILED_PRECONDITION


def test_document_set_merge(client, cleanup):
document_id = 'for-set' + unique_resource_id('-')
document = client.document('i-did-it', document_id)
# Add to clean-up before API request (in case ``set()`` fails).
cleanup(document)

# 0. Make sure the document doesn't exist yet using an option.
option0 = client.write_option(exists=True)
with pytest.raises(NotFound) as exc_info:
document.set({'no': 'way'}, option=option0)

assert exc_info.value.message.startswith(MISSING_DOCUMENT)
assert document_id in exc_info.value.message

# 1. Use ``set()`` to create the document (using an option).
data1 = {'name': 'Sam',
'address': {'city': 'SF',
'state': 'CA'}
}
option1 = client.write_option(exists=False)
write_result1 = document.set(data1, option=option1)
snapshot1 = document.get()
assert snapshot1.to_dict() == data1
# Make sure the update is what created the document.
assert snapshot1.create_time == snapshot1.update_time
assert snapshot1.update_time == write_result1.update_time

# 2. Call ``set()`` again to overwrite (no option).
data2 = {'address.city': 'LA'}
option2 = client.write_option(merge=True)
write_result2 = document.set(data2, option=option2)
snapshot2 = document.get()
assert snapshot2.to_dict() == {'name': 'Sam',
'address': {'city': 'LA',
'state': 'CA'}
}
# Make sure the create time hasn't changed.
assert snapshot2.create_time == snapshot1.create_time
assert snapshot2.update_time == write_result2.update_time


def test_update_document(client, cleanup):
document_id = 'for-update' + unique_resource_id('-')
document = client.document('made', document_id)
Expand All @@ -207,7 +248,7 @@ def test_update_document(client, cleanup):
assert document_id in exc_info.value.message

# 1. Try to update before the document exists (now with an option).
option1 = client.write_option(create_if_missing=False)
option1 = client.write_option(exists=True)
with pytest.raises(NotFound) as exc_info:
document.update({'still': 'not-there'}, option=option1)
assert exc_info.value.message.startswith(MISSING_DOCUMENT)
Expand All @@ -223,7 +264,7 @@ def test_update_document(client, cleanup):
},
'other': True,
}
option2 = client.write_option(create_if_missing=True)
option2 = client.write_option(exists=False)
write_result2 = document.update(data, option=option2)

# 3. Send an update without a field path (no option).
Expand Down
Loading