Optimize queries for prototype lookup, as part of #2126.

CHANGELOG.md

@@ -72,6 +72,8 @@ without arguments starts a full interactive Python console.
   candidates, Builders+ will use list, local search and only global search if no match found.
 - Make `cmd.at_post_cmd()` always run after `cmd.func()`, even when the latter uses delays 
   with yield.
+- Add new `return_iterators` kwarg to `search_prototypes` function in order to prepare for 
+  more paginated handling of prototype returns.
 
 
 ## Evennia 0.9 (2018-2019)

evennia/prototypes/prototypes.py

@@ -9,8 +9,11 @@ import hashlib
 import time
 from ast import literal_eval
 from django.conf import settings
+from django.db.models import Q, Subquery
+from django.core.paginator import Paginator
 from evennia.scripts.scripts import DefaultScript
 from evennia.objects.models import ObjectDB
+from evennia.typeclasses.attributes import Attribute
 from evennia.utils.create import create_script
 from evennia.utils.utils import (
     all_from_module,
@@ -320,7 +323,7 @@ def delete_prototype(prototype_key, caller=None):
     return True
 
 
-def search_prototype(key=None, tags=None, require_single=False):
+def search_prototype(key=None, tags=None, require_single=False, return_iterators=False):
     """
     Find prototypes based on key and/or tags, or all prototypes.
 
@@ -331,11 +334,17 @@ def search_prototype(key=None, tags=None, require_single=False):
             tag category.
         require_single (bool): If set, raise KeyError if the result
             was not found or if there are multiple matches.
+        return_iterators (bool): Optimized return for large numbers of db-prototypes.
+            If set, separate returns of module based prototypes and paginate
+            the db-prototype return.
 
     Return:
-        matches (list): All found prototype dicts. Empty list if
+        matches (list): Default return, all found prototype dicts. Empty list if
             no match was found. Note that if neither `key` nor `tags`
             were given, *all* available prototypes will be returned.
+        list, queryset: If `return_iterators` are found, this is a list of
+            module-based prototypes followed by a *paginated* queryset of
+            db-prototypes.
 
     Raises:
         KeyError: If `require_single` is True and there are 0 or >1 matches.
@@ -387,27 +396,41 @@ def search_prototype(key=None, tags=None, require_single=False):
     if key:
         # exact or partial match on key
         db_matches = (
-            db_matches.filter(db_key=key) or db_matches.filter(db_key__icontains=key)
+            db_matches
-        ).order_by("id")
+            .filter(
-        # return prototype
+                Q(db_key__iexact=key) | Q(db_key__icontains=key))
-    db_prototypes = [dbprot.prototype for dbprot in db_matches]
+            .order_by("id")
-
+        )
-    matches = db_prototypes + module_prototypes
+    # convert to prototype
-    nmatches = len(matches)
+    db_ids = db_matches.values_list("id", flat=True)
-    if nmatches > 1 and key:
+    db_matches = (
-        key = key.lower()
+        Attribute.objects
-        # avoid duplicates if an exact match exist between the two types
+        .filter(scriptdb__pk__in=db_ids, db_key="prototype")
-        filter_matches = [
+        .values_list("db_value", flat=True)
-            mta for mta in matches if mta.get("prototype_key") and mta["prototype_key"] == key
+    )
-        ]
+    if key:
-        if filter_matches and len(filter_matches) < nmatches:
+        matches = list(db_matches) + module_prototypes
-            matches = filter_matches
+        nmatches = len(matches)
-
+        if nmatches > 1:
-    nmatches = len(matches)
+            key = key.lower()
-    if nmatches != 1 and require_single:
+            # avoid duplicates if an exact match exist between the two types
-        raise KeyError("Found {} matching prototypes.".format(nmatches))
+            filter_matches = [
-
+                mta for mta in matches if mta.get("prototype_key") and mta["prototype_key"] == key
-    return matches
+            ]
+            if filter_matches and len(filter_matches) < nmatches:
+                matches = filter_matches
+        nmatches = len(matches)
+        if nmatches != 1 and require_single:
+            raise KeyError("Found {} matching prototypes.".format(nmatches))
+        return matches
+    elif return_iterators:
+        # trying to get the entire set of prototypes - we must paginate
+        # we must paginate the result of trying to fetch the entire set
+        db_pages = Paginator(db_matches, 500)
+        return module_prototypes, db_pages
+    else:
+        # full fetch, no pagination
+        return list(db_matches) + module_prototypes
 
 
 def search_objects_with_prototype(prototype_key):

evennia/prototypes/tests.py

@@ -3,8 +3,10 @@ Unit tests for the prototypes and spawner
 
 """
 
-from random import randint
+from random import randint, sample
 import mock
+import uuid
+from time import time
 from anything import Something
 from django.test.utils import override_settings
 from evennia.utils.test_resources import EvenniaTest
@@ -1073,3 +1075,29 @@ class TestOLCMenu(TestEvMenu):
             ["node_index", "node_index", "node_index"],
         ],
     ]
+
+class PrototypeCrashTest(EvenniaTest):
+
+    # increase this to 1000 for optimization testing
+    num_prototypes = 10
+
+    def create(self, num=None):
+        if not num:
+            num = self.num_prototypes
+        # print(f"Creating {num} additional prototypes...")
+        for x in range(num):
+            prot = {
+                'prototype_key': str(uuid.uuid4()),
+                'some_attributes': [str(uuid.uuid4()) for x in range(10)],
+                'prototype_tags': list(sample(['demo', 'test', 'stuff'], 2)),
+            }
+            protlib.save_prototype(prot)
+
+    def test_prototype_dos(self, *args, **kwargs):
+        num_prototypes = self.num_prototypes
+        for x in range(2):
+            self.create(num_prototypes)
+            # print("Attempting to list prototypes...")
+            start_time = time()
+            self.char1.execute_cmd('spawn/list')
+            # print(f"Prototypes listed in {time()-start_time} seconds.")