optimize queries, mod9 switch to DS

Author: wescpy

Diff for: README.md

@@ -1,7 +1,7 @@
 # Modernizing Google Cloud serverless compute applications
 ### To the latest Cloud services and serverless platforms
 
-This is the corresponding repository to the [Serverless Migration Station](https://bit.ly/3xk2Swi) video series whose goal is to help users on a Google Cloud serverless compute platform modernize to newer Cloud products or other serverless compute platforms. Each modernization migration aims to feature a video, codelab (self-paced, hands-on tutorial), and code samples. The content initially focuses on App Engine and Google's earliest Cloud users.
+This is the corresponding repository to the [Serverless Migration Station](https://bit.ly/3xk2Swi) video series whose goal is to help users on a Google Cloud serverless compute platform modernize to newer Cloud products or other serverless compute platforms. Each modernization migration aims to feature a video, codelab (self-paced, hands-on tutorial), and code samples. The content initially focuses on App Engine and Google's earliest Cloud users. Read more about the [codelabs in this announcement](https://developers.googleblog.com/2021/03/modernizing-your-google-app-engine-applications.html?utm_source=ext&utm_medium=partner&utm_campaign=CDR_wes_aap-serverless_modernizegae_codelabsannounce_201031&utm_content=-) as well as [this one introducing the video series](https://developers.googleblog.com/2021/06/introducing-serverless-migration.html?utm_source=blog&utm_medium=partner&utm_campaign=CDR_wes_aap-serverless_smsintro_201023).
 
 [Google App Engine](https://cloud.google.com/appengine) (Standard) has undergone significant changes between the legacy and next generation platforms. To address this, we've created a set of codelabs (free, online, self-paced, hands-on tutorials) and corresponding videos (when available) to show developers how to perform individual migrations they can apply to modernize their apps for the latest runtimes, with this repo managing the samples from those codelabs. Codelab content typically falls into one of these three topics:
 
@@ -142,7 +142,7 @@ If there is a logical codelab to do immediately after completing one, they will
 - **Module 9 codelab** (TBD): **Migrate a Python 2 Cloud NDB & Cloud Tasks (v1) app to a Python 3 Cloud Datastore & Cloud Tasks (v2) app**
     - **Optional** migrations
         - Migrating to Python 3 is not required but recommended as Python 2 has been sunset
-        - Migrating to Cloud Firestore is *very* optional as Cloud NDB works on 3.x and most importantly, Cloud Firestore requires a completely new GCP project
+        - Migrating to Cloud Datastore is optional as Cloud NDB works on 3.x
     - Python 2
         - START:  [Module 8 code - Cloud Tasks](/mod8-cloudtasks) (2.x)
     - Python 3
@@ -202,7 +202,7 @@ If there is a logical codelab to do immediately after completing one, they will
         - Recommende for small apps or for breaking up large apps into multiple microservices
     - Python 3 only
         - START:  [Module 2 code - Cloud NDB](/mod2b-cloudndb) (3.x)
-        - FINISH: [Module 11 code - Cloud Firestore](/mod11-functions) (3.x)
+        - FINISH: [Module 11 code - Cloud Functions](/mod11-functions) (3.x)
     - RECOMMENDED:
         - Module 7 - add App Engine (push) tasks
     - OTHER OPTIONS (in somewhat priority order):

Diff for: mod0-baseline/main.py

@@ -28,8 +28,7 @@ def store_visit(remote_addr, user_agent):
 
 def fetch_visits(limit):
     'get most recent visits'
-    return (v.to_dict() for v in Visit.query().order(
-            -Visit.timestamp).fetch(limit))
+    return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 class MainHandler(webapp2.RequestHandler):
     'main application (GET) handler'

Diff for: mod1-flask/main.py

@@ -28,8 +28,7 @@ def store_visit(remote_addr, user_agent):
 
 def fetch_visits(limit):
     'get most recent visits'
-    return (v.to_dict() for v in Visit.query().order(
-            -Visit.timestamp).fetch(limit))
+    return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():

Diff for: mod11-functions/main.py

@@ -30,8 +30,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 def visitme(request):
     'main application (GET) handler'

Diff for: mod12-memcache/main.py

@@ -30,8 +30,7 @@ def store_visit(remote_addr, user_agent):
 
 def fetch_visits(limit):
     'get most recent visits'
-    return (v.to_dict() for v in Visit.query().order(
-            -Visit.timestamp).fetch(limit))
+    return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():
@@ -42,7 +41,7 @@ def root():
     visits = memcache.get('visits')
 
     # register visit & run DB query if cache empty or new visitor
-    if not visits or visits[0]['visitor'] != visitor:
+    if not visits or visits[0].visitor != visitor:
         store_visit(ip_addr, usr_agt)
         visits = list(fetch_visits(10))
         memcache.set('visits', visits, HOUR)  # set() not add()

Diff for: mod12b-memcache/main.py

@@ -31,8 +31,7 @@ def store_visit(remote_addr, user_agent):
 
 def fetch_visits(limit):
     'get most recent visits'
-    return (v.to_dict() for v in Visit.query().order(
-            -Visit.timestamp).fetch(limit))
+    return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():
@@ -43,7 +42,7 @@ def root():
     visits = memcache.get('visits')
 
     # register visit & run DB query if cache empty or new visitor
-    if not visits or visits[0]['visitor'] != visitor:
+    if not visits or visits[0].visitor != visitor:
         store_visit(ip_addr, usr_agt)
         visits = list(fetch_visits(10))
         memcache.set('visits', visits, HOUR)  # set() not add()

Diff for: mod13a-memorystore/app.yaml

@@ -20,6 +20,12 @@ handlers:
 - url: /.*
   script: main.app
 
+libraries:
+- name: grpcio
+  version: 1.0.0
+- name: setuptools
+  version: 36.6.0
+
 env_variables:
     REDIS_HOST: 'YOUR_REDIS_HOST'
     REDIS_PORT: 'YOUR_REDIS_PORT'

Diff for: mod13a-memorystore/appengine_config.py

@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pkg_resources
 from google.appengine.ext import vendor
 
 # Set PATH to your libraries folder.
 PATH = 'lib'
 # Add libraries installed in the PATH folder.
 vendor.add(PATH)
+# Add libraries to pkg_resources working set to find the distribution.
+pkg_resources.working_set.add_entry(PATH)

Diff for: mod13a-memorystore/main.py

@@ -38,8 +38,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():
@@ -51,7 +50,7 @@ def root():
     visits = pickle.loads(rsp) if rsp else None
 
     # register visit & run DB query if cache empty or new visitor
-    if not visits or visits[0]['visitor'] != visitor:
+    if not visits or visits[0].visitor != visitor:
         store_visit(ip_addr, usr_agt)
         visits = list(fetch_visits(10))
         REDIS.set('visits', pickle.dumps(visits), ex=HOUR)

Diff for: mod13b-memorystore/main.py

@@ -38,8 +38,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():
@@ -51,7 +50,7 @@ def root():
     visits = pickle.loads(rsp) if rsp else None
 
     # register visit & run DB query if cache empty or new visitor
-    if not visits or visits[0]['visitor'] != visitor:
+    if not visits or visits[0].visitor != visitor:
         store_visit(ip_addr, usr_agt)
         visits = list(fetch_visits(10))
         REDIS.set('visits', pickle.dumps(visits), ex=HOUR)

Diff for: mod1b-flask/main.py

@@ -30,8 +30,7 @@ def store_visit(remote_addr, user_agent):
 
 def fetch_visits(limit):
     'get most recent visits'
-    return (v.to_dict() for v in Visit.query().order(
-            -Visit.timestamp).fetch(limit))
+    return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():

Diff for: mod2a-cloudndb/main.py

@@ -31,8 +31,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():

Diff for: mod2b-cloudndb/main.py

@@ -31,8 +31,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():

Diff for: mod2b-cloudndb/requirements.txt

@@ -1,2 +1,2 @@
 flask
-google-cloud-ndb==1.11.1
+google-cloud-ndb

Diff for: mod4a-rundocker/main.py

@@ -31,8 +31,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():

Diff for: mod4a-rundocker/requirements.txt

@@ -1,3 +1,3 @@
-gunicorn==19.10.0
+gunicorn
 flask
 google-cloud-ndb==1.11.1

Diff for: mod5-runbldpks/main.py

@@ -31,8 +31,7 @@ def store_visit(remote_addr, user_agent):
 def fetch_visits(limit):
     'get most recent visits'
     with ds_client.context():
-        return (v.to_dict() for v in Visit.query().order(
-                -Visit.timestamp).fetch(limit))
+        return Visit.query().order(-Visit.timestamp).fetch(limit)
 
 @app.route('/')
 def root():

Diff for: mod5-runbldpks/requirements.txt

@@ -1,3 +1,3 @@
-gunicor
+gunicorn
 flask
 google-cloud-ndb

Diff for: mod7-gaetasks/main.py

@@ -37,7 +37,7 @@ def fetch_visits(limit):
     oldest_str = time.ctime(oldest)
     logging.info('Delete entities older than %s' % oldest_str)
     taskqueue.add(url='/trim', params={'oldest': oldest})
-    return (v.to_dict() for v in data), oldest_str
+    return data, oldest_str
 
 @app.route('/trim', methods=['POST'])
 def trim():

Diff for: mod7b-gaetasks/main.py

@@ -38,7 +38,7 @@ def fetch_visits(limit):
     oldest_str = time.ctime(oldest)
     logging.info('Delete entities older than %s' % oldest_str)
     taskqueue.add(url='/trim', params={'oldest': oldest})
-    return (v.to_dict() for v in data), oldest_str
+    return data, oldest_str
 
 @app.route('/trim', methods=['POST'])
 def trim():

Diff for: mod8-cloudtasks/main.py

@@ -56,7 +56,7 @@ def fetch_visits(limit):
         }
     }
     ts_client.create_task(parent=QUEUE_PATH, task=task)
-    return (v.to_dict() for v in data), oldest_str
+    return data, oldest_str
 
 @app.route('/trim', methods=['POST'])
 def trim():

Diff for: mod9-py3fstasks/main.py

@@ -52,7 +52,7 @@ def fetch_visits(limit):
     'get most recent visits & add task to delete older visits'
     query = ds_client.query(kind='Visit')
     query.order = ['-timestamp']
-    visits = query.fetch(limit=limit)
+    visits = list(query.fetch(limit=limit))
     oldest = time.mktime(visits[-1]['timestamp'].timetuple())
     oldest_str = time.ctime(oldest)
     print('Delete entities older than %s' % oldest_str)
@@ -69,22 +69,19 @@ def fetch_visits(limit):
         ts_client.create_task(parent=QUEUE_PATH, task=task)
     return visits, oldest_str
 
-def _delete_docs(visits):
-    'app-internal generator deleting old FS visit documents'
-    for visit in visits:
-        visit.reference.delete()
-        yield visit.id
-
 @app.route('/trim', methods=['POST'])
 def trim():
     '(push) task queue handler to delete oldest visits'
     oldest = float(request.get_json().get('oldest'))
-    query = ds_client.collection('Visit')
-    visits = query.where('timestamp', '<',
-            datetime.fromtimestamp(oldest)).stream()
-    dlist = ', '.join(str(v_id) for v_id in _delete_docs(visits))
-    if dlist:
-        print('Deleting %d entities: %s' % (dlist.count(',')+1, dlist))
+    query = ds_client.query(kind='Visit')
+    query.add_filter('timestamp', '<', datetime.fromtimestamp(oldest))
+    query.keys_only()
+    keys = list(visit.key for visit in query.fetch())
+    nkeys = len(keys)
+    if nkeys:
+        print('Deleting %d entities: %s' % (
+                nkeys, ', '.join(str(k.id) for k in keys)))
+        ds_client.delete_multi(keys)
     else:
         print('No entities older than: %s' % time.ctime(oldest))
     return ''   # need to return SOME string w/200

Diff for: mod9-py3fstasks/requirements.txt

@@ -1,3 +1,3 @@
 flask
-google-cloud-firestore==2.3.4
-google-cloud-tasks==2.7.1
+google-cloud-datastore
+google-cloud-tasks