Implement new EvMore functionality. Resolves #2126

CHANGELOG.md

@@ -70,8 +70,10 @@ without arguments starts a full interactive Python console.
   candidates, Builders+ will use list, local search and only global search if no match found.
 - Make `cmd.at_post_cmd()` always run after `cmd.func()`, even when the latter uses delays 
   with yield.
-- Add new `return_iterators` kwarg to `search_prototypes` function in order to prepare for 
+- `EvMore` support for db queries and django paginators as well as easier to override for custom
-  more paginated handling of prototype returns.
+  pagination (e.g. to create EvTables for every page instead of splittine one table)
+- Using `EvMore pagination`, dramatically improves performance of `spawn/list` and `scripts` listings 
+  (100x speed increase for displaying 1000+ prototypes/scripts).
   
 
 ## Evennia 0.9 (2018-2019)

evennia/commands/default/building.py

@@ -3076,9 +3076,9 @@ class CmdScript(COMMAND_DEFAULT_CLASS):
                 result.append("No scripts defined on %s." % obj.get_display_name(caller))
             elif not self.switches:
                 # view all scripts
-                from evennia.commands.default.system import format_script_list
+                from evennia.commands.default.system import ScriptEvMore
-
+                ScriptEvMore(self.caller, scripts.order_by("id"), session=self.session)
-                result.append(format_script_list(scripts))
+                return
             elif "start" in self.switches:
                 num = sum([obj.scripts.start(script.key) for script in scripts])
                 result.append("%s scripts started on %s." % (num, obj.get_display_name(caller)))
@@ -3285,6 +3285,7 @@ class CmdSpawn(COMMAND_DEFAULT_CLASS):
 
       spawn/search [prototype_keykey][;tag[,tag]]
       spawn/list [tag, tag, ...]
+      spawn/list modules    - list only module-based prototypes
       spawn/show [<prototype_key>]
       spawn/update <prototype_key>
 

evennia/commands/default/system.py

@@ -16,6 +16,7 @@ import twisted
 import time
 
 from django.conf import settings
+from django.core.paginator import Paginator
 from evennia.server.sessionhandler import SESSIONS
 from evennia.scripts.models import ScriptDB
 from evennia.objects.models import ObjectDB
@@ -406,59 +407,71 @@ class CmdPy(COMMAND_DEFAULT_CLASS):
         )
 
 
-# helper function. Kept outside so it can be imported and run
+class ScriptEvMore(EvMore):
-# by other commands.
+    """
+    Listing 1000+ Scripts can be very slow and memory-consuming. So
+    we use this custom EvMore child to build en EvTable only for
+    each page of the list.
 
+    """
 
-def format_script_list(scripts):
+    def init_pages(self, scripts):
-    """Takes a list of scripts and formats the output."""
+        """Prepare the script list pagination"""
-    if not scripts:
+        script_pages = Paginator(scripts, max(1, int(self.height / 2)))
-        return "<No scripts>"
+        super().init_pages(script_pages)
 
-    table = EvTable(
+    def page_formatter(self, scripts):
-        "|wdbref|n",
+        """Takes a page of scripts and formats the output
-        "|wobj|n",
+        into an EvTable."""
-        "|wkey|n",
-        "|wintval|n",
-        "|wnext|n",
-        "|wrept|n",
-        "|wdb",
-        "|wtypeclass|n",
-        "|wdesc|n",
-        align="r",
-        border="tablecols",
-    )
 
-    for script in scripts:
+        if not scripts:
+            return "<No scripts>"
 
-        nextrep = script.time_until_next_repeat()
+        table = EvTable(
-        if nextrep is None:
+            "|wdbref|n",
-            nextrep = "PAUSED" if script.db._paused_time else "--"
+            "|wobj|n",
-        else:
+            "|wkey|n",
-            nextrep = "%ss" % nextrep
+            "|wintval|n",
-
+            "|wnext|n",
-        maxrepeat = script.repeats
+            "|wrept|n",
-        remaining = script.remaining_repeats() or 0
+            "|wdb",
-        if maxrepeat:
+            "|wtypeclass|n",
-            rept = "%i/%i" % (maxrepeat - remaining, maxrepeat)
+            "|wdesc|n",
-        else:
+            align="r",
-            rept = "-/-"
+            border="tablecols",
-
+            width=self.width
-        table.add_row(
-            script.id,
-            f"{script.obj.key}({script.obj.dbref})"
-            if (hasattr(script, "obj") and script.obj)
-            else "<Global>",
-            script.key,
-            script.interval if script.interval > 0 else "--",
-            nextrep,
-            rept,
-            "*" if script.persistent else "-",
-            script.typeclass_path.rsplit(".", 1)[-1],
-            crop(script.desc, width=20),
         )
 
-    return "%s" % table
+        for script in scripts:
+
+            nextrep = script.time_until_next_repeat()
+            if nextrep is None:
+                nextrep = "PAUSED" if script.db._paused_time else "--"
+            else:
+                nextrep = "%ss" % nextrep
+
+            maxrepeat = script.repeats
+            remaining = script.remaining_repeats() or 0
+            if maxrepeat:
+                rept = "%i/%i" % (maxrepeat - remaining, maxrepeat)
+            else:
+                rept = "-/-"
+
+            table.add_row(
+                script.id,
+                f"{script.obj.key}({script.obj.dbref})"
+                if (hasattr(script, "obj") and script.obj)
+                else "<Global>",
+                script.key,
+                script.interval if script.interval > 0 else "--",
+                nextrep,
+                rept,
+                "*" if script.persistent else "-",
+                script.typeclass_path.rsplit(".", 1)[-1],
+                crop(script.desc, width=20),
+            )
+
+        return str(table)
 
 
 class CmdScripts(COMMAND_DEFAULT_CLASS):
@@ -547,7 +560,7 @@ class CmdScripts(COMMAND_DEFAULT_CLASS):
                 caller.msg(string)
             else:
                 # multiple matches.
-                EvMore(caller, scripts, page_formatter=format_script_list)
+                ScriptEvMore(caller, scripts, session=self.session)
                 caller.msg("Multiple script matches. Please refine your search")
         elif self.switches and self.switches[0] in ("validate", "valid", "val"):
             # run validation on all found scripts
@@ -557,7 +570,7 @@ class CmdScripts(COMMAND_DEFAULT_CLASS):
             caller.msg(string)
         else:
             # No stopping or validation. We just want to view things.
-            EvMore(caller, scripts, page_formatter=format_script_list)
+            ScriptEvMore(caller, scripts.order_by('id'), session=self.session)
 
 
 class CmdObjects(COMMAND_DEFAULT_CLASS):

evennia/commands/default/tests.py

@@ -1239,10 +1239,11 @@ class TestBuilding(CommandTest):
         )
 
     def test_spawn(self):
-        def getObject(commandTest, objKeyStr):
+
+        def get_object(commandTest, obj_key):
             # A helper function to get a spawned object and
             # check that it exists in the process.
-            query = search_object(objKeyStr)
+            query = search_object(obj_key)
             commandTest.assertIsNotNone(query)
             commandTest.assertTrue(bool(query))
             obj = query[0]
@@ -1271,7 +1272,7 @@ class TestBuilding(CommandTest):
         )
 
         self.call(building.CmdSpawn(), "/search ", "Key ")
-        self.call(building.CmdSpawn(), "/search test;test2", "")
+        self.call(building.CmdSpawn(), "/search test;test2", "No prototypes found.")
 
         self.call(
             building.CmdSpawn(),
@@ -1281,11 +1282,11 @@ class TestBuilding(CommandTest):
         )
 
         self.call(building.CmdSpawn(), "/list", "Key ")
-
         self.call(building.CmdSpawn(), "testprot", "Spawned Test Char")
-        # Tests that the spawned object's location is the same as the caharacter's location, since
+
+        # Tests that the spawned object's location is the same as the character's location, since
         # we did not specify it.
-        testchar = getObject(self, "Test Char")
+        testchar = get_object(self, "Test Char")
         self.assertEqual(testchar.location, self.char1.location)
         testchar.delete()
 
@@ -1302,7 +1303,7 @@ class TestBuilding(CommandTest):
             "'key':'goblin', 'location':'%s'}" % spawnLoc.dbref,
             "Spawned goblin",
         )
-        goblin = getObject(self, "goblin")
+        goblin = get_object(self, "goblin")
         # Tests that the spawned object's type is a DefaultCharacter.
         self.assertIsInstance(goblin, DefaultCharacter)
         self.assertEqual(goblin.location, spawnLoc)
@@ -1321,7 +1322,7 @@ class TestBuilding(CommandTest):
         # Tests "spawn <prototype_name>"
         self.call(building.CmdSpawn(), "testball", "Spawned Ball")
 
-        ball = getObject(self, "Ball")
+        ball = get_object(self, "Ball")
         self.assertEqual(ball.location, self.char1.location)
         self.assertIsInstance(ball, DefaultObject)
         ball.delete()
@@ -1331,7 +1332,7 @@ class TestBuilding(CommandTest):
         self.call(
             building.CmdSpawn(), "/n 'BALL'", "Spawned Ball"
         )  # /n switch is abbreviated form of /noloc
-        ball = getObject(self, "Ball")
+        ball = get_object(self, "Ball")
         self.assertIsNone(ball.location)
         ball.delete()
 
@@ -1350,7 +1351,7 @@ class TestBuilding(CommandTest):
             % spawnLoc.dbref,
             "Spawned Ball",
         )
-        ball = getObject(self, "Ball")
+        ball = get_object(self, "Ball")
         self.assertEqual(ball.location, spawnLoc)
         ball.delete()
 

evennia/prototypes/prototypes.py

@@ -391,24 +391,37 @@ def search_prototype(key=None, tags=None, require_single=False, return_iterators
         # exact match on tag(s)
         tags = make_iter(tags)
         tag_categories = ["db_prototype" for _ in tags]
-        db_matches = DbPrototype.objects.get_by_tag(tags, tag_categories)
+        db_matches = DbPrototype.objects.get_by_tag(
+            tags, tag_categories)
     else:
-        db_matches = DbPrototype.objects.all().order_by("id")
+        db_matches = DbPrototype.objects.all()
 
     if key:
         # exact or partial match on key
-        db_matches = (
+        exact_match = (
             db_matches
             .filter(
-                Q(db_key__iexact=key) | Q(db_key__icontains=key))
+                Q(db_key__iexact=key))
-            .order_by("id")
+            .order_by("db_key")
         )
+        if not exact_match:
+            # try with partial match instead
+            db_matches = (
+                db_matches
+                .filter(
+                    Q(db_key__icontains=key))
+                .order_by("db_key")
+            )
+        else:
+            db_matches = exact_match
+
     # convert to prototype
     db_ids = db_matches.values_list("id", flat=True)
     db_matches = (
         Attribute.objects
         .filter(scriptdb__pk__in=db_ids, db_key="prototype")
         .values_list("db_value", flat=True)
+        .order_by("scriptdb__db_key")
     )
     if key and require_single:
         nmodules = len(module_prototypes)
@@ -419,10 +432,9 @@ def search_prototype(key=None, tags=None, require_single=False, return_iterators
     if return_iterators:
         # trying to get the entire set of prototypes - we must paginate
         # the result instead of trying to fetch the entire set at once
-        db_pages = Paginator(db_matches, 20)
+        return db_matches, module_prototypes
-        return module_prototypes, db_pages
     else:
-        # full fetch, no pagination
+        # full fetch, no pagination (compatibility mode)
         return list(db_matches) + module_prototypes
 
 
@@ -451,18 +463,45 @@ class PrototypeEvMore(EvMore):
         """Store some extra properties on the EvMore class"""
         self.show_non_use = kwargs.pop("show_non_use", False)
         self.show_non_edit = kwargs.pop("show_non_edit", False)
-
-        # set up table width
-        width = settings.CLIENT_DEFAULT_WIDTH
-        if not session:
-            # fall back to the first session
-            session = caller.sessions.all()[0]
-        if session:
-            width = session.protocol_flags.get("SCREENWIDTH", {0: width})[0]
-        self.width = width
-
         super().__init__(caller, *args, session=session, **kwargs)
 
+    def init_pages(self, inp):
+        """
+        This will be initialized with a tuple (mod_prototype_list, paginated_db_query)
+        and we must handle these separately since they cannot be paginated in the same
+        way. We will build the prototypes so that the db-prototypes come first (they
+        are likely the most volatile), followed by the mod-prototypes.
+        """
+        dbprot_query, modprot_list = inp
+        # set the number of entries per page to half the reported height of the screen
+        # to account for long descs etc
+        dbprot_paged = Paginator(dbprot_query, max(1, int(self.height / 2)))
+
+        # we separate the different types of data, so we track how many pages there are
+        # of each.
+        n_mod = len(modprot_list)
+        self._npages_mod = n_mod // self.height + (0 if n_mod % self.height == 0 else 1)
+        self._db_count = dbprot_paged.count
+        self._npages_db = dbprot_paged.num_pages if self._db_count > 0 else 0
+        # total number of pages
+        self._npages = self._npages_mod + self._npages_db
+        self._data = (dbprot_paged, modprot_list)
+        self._paginator = self.prototype_paginator
+
+    def prototype_paginator(self, pageno):
+        """
+        The listing is separated in db/mod prototypes, so we need to figure out which
+        one to pick based on the page number. Also, pageno starts from 0.
+        """
+        dbprot_pages, modprot_list = self._data
+
+        if self._db_count and pageno < self._npages_db:
+            return dbprot_pages.page(pageno + 1)
+        else:
+            # get the correct slice, adjusted for the db-prototypes
+            pageno = max(0, pageno - self._npages_db)
+            return modprot_list[pageno * self.height: pageno * self.height + self.height]
+
     def page_formatter(self, page):
         """Input is a queryset page from django.Paginator"""
         caller = self._caller
@@ -470,7 +509,15 @@ class PrototypeEvMore(EvMore):
         # get use-permissions of readonly attributes (edit is always False)
         display_tuples = []
 
-        print("page", page)
+        table = EvTable(
+            "|wKey|n",
+            "|wSpawn/Edit|n",
+            "|wTags|n",
+            "|wDesc|n",
+            border="tablecols",
+            crop=True,
+            width=self.width
+        )
 
         for prototype in page:
             lock_use = caller.locks.check_lockstring(
@@ -490,36 +537,24 @@ class PrototypeEvMore(EvMore):
             for ptag in prototype.get("prototype_tags", []):
                 if is_iter(ptag):
                     if len(ptag) > 1:
-                        ptags.append("{} (category: {})".format(ptag[0], ptag[1]))
+                        ptags.append("{}".format(ptag[0]))
                     else:
                         ptags.append(ptag[0])
                 else:
                     ptags.append(str(ptag))
 
-            display_tuples.append(
+            table.add_row(
-                (
+                prototype.get("prototype_key", "<unset>"),
-                    prototype.get("prototype_key", "<unset>"),
+                "{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"),
-                    "{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"),
+                ", ".join(list(set(ptags))),
-                    "\n".join(list(set(ptags))),
+                prototype.get("prototype_desc", "<unset>"),
-                    prototype.get("prototype_desc", "<unset>"),
-                )
             )
 
-        if not display_tuples:
-            return ""
-
-        table = []
-        for i in range(len(display_tuples[0])):
-            table.append([str(display_tuple[i]) for display_tuple in display_tuples])
-        table = EvTable("Key", "Spawn/Edit", "Tags", "Desc", table=table, crop=True, width=self.width)
-        table.reformat_column(0, width=22)
-        table.reformat_column(1, width=9, align="c")
-        table.reformat_column(2)
-        table.reformat_column(3)
         return str(table)
 
 
-def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_edit=True, session=None):
+def list_prototypes(caller, key=None, tags=None, show_non_use=False,
+                    show_non_edit=True, session=None):
     """
     Collate a list of found prototypes based on search criteria and access.
 
@@ -532,33 +567,23 @@ def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_ed
         session (Session, optional): If given, this is used for display formatting.
     Returns:
         PrototypeEvMore: An EvMore subclass optimized for prototype listings.
-        None: If a `key` was given and no matches was found. In this case the caller
+        None: If no matches were found. In this case the caller has already been notified.
-            has already been notified.
 
     """
     # this allows us to pass lists of empty strings
     tags = [tag for tag in make_iter(tags) if tag]
 
-    if key is not None:
+    dbprot_query, modprot_list = search_prototype(key, tags, return_iterators=True)
-        matches = search_prototype(key, tags)
-        if not matches:
-            caller.msg("No prototypes found.", session=session)
-            return None
-        if len(matches) < 2:
-            matches = [matches]
-        # get specific prototype (one value or exception)
-        return PrototypeEvMore(caller, matches,
-                               session=session,
-                               show_non_use=show_non_use,
-                               show_non_edit=show_non_edit)
-    else:
-        # list all
-        # get prototypes for readonly and db-based prototypes
-        module_prots, db_prots = search_prototype(key, tags, return_iterators=True)
-        return PrototypeEvMore(caller, db_prots,
-                               session=session,
-                               show_non_use=show_non_use, show_non_edit=show_non_edit)
 
+    if not dbprot_query and not modprot_list:
+        caller.msg("No prototypes found.", session=session)
+        return None
+
+    # get specific prototype (one value or exception)
+    return PrototypeEvMore(caller, (dbprot_query, modprot_list),
+                           session=session,
+                           show_non_use=show_non_use,
+                           show_non_edit=show_non_edit)
 
 def validate_prototype(
     prototype, protkey=None, protparents=None, is_prototype_base=True, strict=True, _flags=None

evennia/server/portal/amp_server.py

@@ -159,7 +159,7 @@ class AMPServerProtocol(amp.AMPMultiConnectionProtocol):
 
         """
         # start the Server
-        print("Portal starting server ... {}".format(server_twistd_cmd))
+        print("Portal starting server ... ")
         process = None
         with open(settings.SERVER_LOG_FILE, "a") as logfile:
             # we link stdout to a file in order to catch

evennia/utils/evmore.py

@@ -461,7 +461,7 @@ class EvMore(object):
         elif not isinstance(inp, str):
             # anything else not a str
             self.init_iterable(inp)
-            self._paginator = self.paginator_index
+            self._paginator = self.paginator_slice
         elif "\f" in inp:
             # string with \f line-break markers in it
             self.init_f_str(inp)
@@ -476,7 +476,7 @@ class EvMore(object):
         Paginator. The data operated upon is in `self._data`.
 
         Args:
-            pageno (int): The page number to view, from 1...N
+            pageno (int): The page number to view, from 0...N-1
         Returns:
             str: The page to display (without any decorations, those are added
                 by EvMore).