[datastore] fix: handle datastore eventual consistency (#3448)

* [datastore] fix: handle datastore eventual consistency

fixes #2884

* fix flask_app_test too

datastore/cloud-ndb/flask_app_test.py

@@ -14,8 +14,9 @@
 
 import uuid
 
-import pytest
 
+import backoff
+import pytest
 from google.cloud import ndb
 
 import flask_app
@@ -39,9 +40,13 @@ def test_index(test_book):
     flask_app.app.testing = True
     client = flask_app.app.test_client()
 
-    r = client.get('/')
-    assert r.status_code == 200
-    assert test_book.title in r.data.decode('utf-8')
+    @backoff.on_exception(backoff.expo, AssertionError, max_time=60)
+    def eventually_consistent_test():
+        r = client.get('/')
+        assert r.status_code == 200
+        assert test_book.title in r.data.decode('utf-8')
+
+    eventually_consistent_test()
 
 
 def test_ndb_wsgi_middleware():

datastore/cloud-ndb/quickstart_test.py

@@ -14,6 +14,7 @@
 
 import uuid
 
+import backoff
 import pytest
 
 import quickstart
@@ -34,6 +35,10 @@ def test_book():
 
 
 def test_quickstart(capsys, test_book):
-    quickstart.list_books()
-    out, _ = capsys.readouterr()
-    assert test_book.title in out
+    @backoff.on_exception(backoff.expo, AssertionError, max_time=60)
+    def eventually_consistent_test():
+        quickstart.list_books()
+        out, _ = capsys.readouterr()
+        assert test_book.title in out
+
+    eventually_consistent_test()

datastore/cloud-ndb/requirements-test.txt

@@ -1 +1,2 @@
+backoff==1.10.0
 pytest==5.3.2