Annikka/evennia
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Optimize EvMore for prototypes, as per #2126

Browse source
This commit is contained in:
Griatch 2020-09-09 10:06:59 +02:00
parent a1410ef30f
commit d607bbb0fc
4 changed files with 330 additions and 215 deletions
Download patch file Download diff file Expand all files Collapse all files

14
evennia/commands/default/building.py
View file

@ -3476,16 +3476,16 @@ class CmdSpawn(COMMAND_DEFAULT_CLASS):
elif query: elif query:
self.caller.msg(f"No prototype named '{query}' was found.") self.caller.msg(f"No prototype named '{query}' was found.")
else: else:
self.caller.msg(f"No prototypes found.") self.caller.msg("No prototypes found.")
def _list_prototypes(self, key=None, tags=None): def _list_prototypes(self, key=None, tags=None):
"""Display prototypes as a list, optionally limited by key/tags. """ """Display prototypes as a list, optionally limited by key/tags. """
table = protlib.list_prototypes(self.caller, key=key, tags=tags) protlib.list_prototypes(self.caller, key=key, tags=tags)
if not table: # if not table:
return True # return True
EvMore( # EvMore(
self.caller, str(table), exit_on_lastpage=True, justify_kwargs=False, # self.caller, str(table), exit_on_lastpage=True, justify_kwargs=False,
) # )
@interactive @interactive
def _update_existing_objects(self, caller, prototype_key, quiet=False): def _update_existing_objects(self, caller, prototype_key, quiet=False):

183
evennia/prototypes/prototypes.py
View file

@ -15,6 +15,7 @@ from evennia.scripts.scripts import DefaultScript
from evennia.objects.models import ObjectDB from evennia.objects.models import ObjectDB
from evennia.typeclasses.attributes import Attribute from evennia.typeclasses.attributes import Attribute
from evennia.utils.create import create_script from evennia.utils.create import create_script
from evennia.utils.evmore import EvMore
from evennia.utils.utils import ( from evennia.utils.utils import (
all_from_module, all_from_module,
make_iter, make_iter,
@ -393,6 +394,7 @@ def search_prototype(key=None, tags=None, require_single=False, return_iterators
db_matches = DbPrototype.objects.get_by_tag(tags, tag_categories) db_matches = DbPrototype.objects.get_by_tag(tags, tag_categories)
else: else:
db_matches = DbPrototype.objects.all().order_by("id") db_matches = DbPrototype.objects.all().order_by("id")
if key: if key:
# exact or partial match on key # exact or partial match on key
db_matches = ( db_matches = (
@ -425,8 +427,8 @@ def search_prototype(key=None, tags=None, require_single=False, return_iterators
return matches return matches
elif return_iterators: elif return_iterators:
# trying to get the entire set of prototypes - we must paginate # trying to get the entire set of prototypes - we must paginate
# we must paginate the result of trying to fetch the entire set # the result instead of trying to fetch the entire set at once
db_pages = Paginator(db_matches, 500) db_pages = Paginator(db_matches, 20)
return module_prototypes, db_pages return module_prototypes, db_pages
else: else:
# full fetch, no pagination # full fetch, no pagination
@ -447,6 +449,74 @@ def search_objects_with_prototype(prototype_key):
return ObjectDB.objects.get_by_tag(key=prototype_key, category=PROTOTYPE_TAG_CATEGORY) return ObjectDB.objects.get_by_tag(key=prototype_key, category=PROTOTYPE_TAG_CATEGORY)
class PrototypeEvMore(EvMore):
"""
Listing 1000+ prototypes can be very slow. So we customize EvMore to
display an EvTable per paginated page rather than to try creating an
EvTable for the entire dataset and then paginate it.
"""
def __init__(self, *args, **kwargs):
"""Store some extra properties on the EvMore class"""
self.show_non_use = kwargs.pop("show_non_use", False)
self.show_non_edit = kwargs.pop("show_non_edit", False)
super().__init__(*args, **kwargs)
def page_formatter(self, page):
"""Input is a queryset page from django.Paginator"""
caller = self._caller
# get use-permissions of readonly attributes (edit is always False)
display_tuples = []
for prototype in page:
lock_use = caller.locks.check_lockstring(
caller, prototype.get("prototype_locks", ""), access_type="spawn", default=True
)
if not self.show_non_use and not lock_use:
continue
if prototype.get("prototype_key", "") in _MODULE_PROTOTYPES:
lock_edit = False
else:
lock_edit = caller.locks.check_lockstring(
caller, prototype.get("prototype_locks", ""), access_type="edit", default=True
)
if not self.show_non_edit and not lock_edit:
continue
ptags = []
for ptag in prototype.get("prototype_tags", []):
if is_iter(ptag):
if len(ptag) > 1:
ptags.append("{} (category: {}".format(ptag[0], ptag[1]))
else:
ptags.append(ptag[0])
else:
ptags.append(str(ptag))
display_tuples.append(
(
prototype.get("prototype_key", "<unset>"),
prototype.get("prototype_desc", "<unset>"),
"{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"),
",".join(ptags),
)
)
if not display_tuples:
return ""
table = []
width = 78
for i in range(len(display_tuples[0])):
table.append([str(display_tuple[i]) for display_tuple in display_tuples])
table = EvTable("Key", "Desc", "Spawn/Edit", "Tags", table=table, crop=True, width=width)
table.reformat_column(0, width=22)
table.reformat_column(1, width=29)
table.reformat_column(2, width=11, align="c")
table.reformat_column(3, width=16)
return str(table)
def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_edit=True): def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_edit=True):
""" """
Collate a list of found prototypes based on search criteria and access. Collate a list of found prototypes based on search criteria and access.
@ -465,57 +535,66 @@ def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_ed
# this allows us to pass lists of empty strings # this allows us to pass lists of empty strings
tags = [tag for tag in make_iter(tags) if tag] tags = [tag for tag in make_iter(tags) if tag]
# get prototypes for readonly and db-based prototypes if key is not None:
prototypes = search_prototype(key, tags) # get specific prototype (one value or exception)
return PrototypeEvMore(caller, [search_prototype(key, tags)],
show_non_use=show_non_use,
show_non_edit=show_non_edit)
else:
# list all
# get prototypes for readonly and db-based prototypes
module_prots, db_prots = search_prototype(key, tags, return_iterators=True)
return PrototypeEvMore(caller, db_prots,
show_non_use=show_non_use, show_non_edit=show_non_edit)
# get use-permissions of readonly attributes (edit is always False) # # get use-permissions of readonly attributes (edit is always False)
display_tuples = [] # display_tuples = []
for prototype in sorted(prototypes, key=lambda d: d.get("prototype_key", "")): # for prototype in sorted(prototypes, key=lambda d: d.get("prototype_key", "")):
lock_use = caller.locks.check_lockstring( # lock_use = caller.locks.check_lockstring(
caller, prototype.get("prototype_locks", ""), access_type="spawn", default=True # caller, prototype.get("prototype_locks", ""), access_type="spawn", default=True
) # )
if not show_non_use and not lock_use: # if not show_non_use and not lock_use:
continue # continue
if prototype.get("prototype_key", "") in _MODULE_PROTOTYPES: # if prototype.get("prototype_key", "") in _MODULE_PROTOTYPES:
lock_edit = False # lock_edit = False
else: # else:
lock_edit = caller.locks.check_lockstring( # lock_edit = caller.locks.check_lockstring(
caller, prototype.get("prototype_locks", ""), access_type="edit", default=True # caller, prototype.get("prototype_locks", ""), access_type="edit", default=True
) # )
if not show_non_edit and not lock_edit: # if not show_non_edit and not lock_edit:
continue # continue
ptags = [] # ptags = []
for ptag in prototype.get("prototype_tags", []): # for ptag in prototype.get("prototype_tags", []):
if is_iter(ptag): # if is_iter(ptag):
if len(ptag) > 1: # if len(ptag) > 1:
ptags.append("{} (category: {}".format(ptag[0], ptag[1])) # ptags.append("{} (category: {}".format(ptag[0], ptag[1]))
else: # else:
ptags.append(ptag[0]) # ptags.append(ptag[0])
else: # else:
ptags.append(str(ptag)) # ptags.append(str(ptag))
#
display_tuples.append( # display_tuples.append(
( # (
prototype.get("prototype_key", "<unset>"), # prototype.get("prototype_key", "<unset>"),
prototype.get("prototype_desc", "<unset>"), # prototype.get("prototype_desc", "<unset>"),
"{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"), # "{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"),
",".join(ptags), # ",".join(ptags),
) # )
) # )
#
if not display_tuples: # if not display_tuples:
return "" # return ""
#
table = [] # table = []
width = 78 # width = 78
for i in range(len(display_tuples[0])): # for i in range(len(display_tuples[0])):
table.append([str(display_tuple[i]) for display_tuple in display_tuples]) # table.append([str(display_tuple[i]) for display_tuple in display_tuples])
table = EvTable("Key", "Desc", "Spawn/Edit", "Tags", table=table, crop=True, width=width) # table = EvTable("Key", "Desc", "Spawn/Edit", "Tags", table=table, crop=True, width=width)
table.reformat_column(0, width=22) # table.reformat_column(0, width=22)
table.reformat_column(1, width=29) # table.reformat_column(1, width=29)
table.reformat_column(2, width=11, align="c") # table.reformat_column(2, width=11, align="c")
table.reformat_column(3, width=16) # table.reformat_column(3, width=16)
return table # return table
def validate_prototype( def validate_prototype(

2
evennia/prototypes/tests.py
View file

@ -1100,6 +1100,6 @@ class PrototypeCrashTest(EvenniaTest):
for x in range(2): for x in range(2):
self.create(num_prototypes) self.create(num_prototypes)
# print("Attempting to list prototypes...") # print("Attempting to list prototypes...")
start_time = time() # start_time = time()
self.char1.execute_cmd('spawn/list') self.char1.execute_cmd('spawn/list')
# print(f"Prototypes listed in {time()-start_time} seconds.") # print(f"Prototypes listed in {time()-start_time} seconds.")

346
evennia/utils/evmore.py
View file

@ -176,27 +176,30 @@ class EvMore(object):
the caller when the more page exits. Note that this will be using whatever the caller when the more page exits. Note that this will be using whatever
cmdset the user had *before* the evmore pager was activated (so none of cmdset the user had *before* the evmore pager was activated (so none of
the evmore commands will be available when this is run). the evmore commands will be available when this is run).
page_formatter (callable, optional): If given, this function will be passed the
contents of each extracted page. This is useful when paginating
data consisting something other than a string or a list of strings. Especially
queryset data is likely to always need this argument specified. Note however,
that all size calculations assume this function to return one single line
per element on the page!
kwargs (any, optional): These will be passed on to the `caller.msg` method. kwargs (any, optional): These will be passed on to the `caller.msg` method.
Examples: Examples:
Basic use:
```
super_long_text = " ... " super_long_text = " ... "
EvMore(caller, super_long_text) EvMore(caller, super_long_text)
```
Paginator
```
from django.core.paginator import Paginator from django.core.paginator import Paginator
query = ObjectDB.objects.all() query = ObjectDB.objects.all()
pages = Paginator(query, 10) # 10 objs per page pages = Paginator(query, 10) # 10 objs per page
EvMore(caller, pages) # will repr() each object per line, 10 to a page EvMore(caller, pages)
```
multi_page_table = [ [[..],[..]], ...] Every page an EvTable
EvMore(caller, multi_page_table, use_evtable=True, ```
evtable_args=("Header1", "Header2"), from evennia import EvTable
evtable_kwargs={"align": "r", "border": "tablecols"}) def _to_evtable(page):
table = ... # convert page to a table
return EvTable(*headers, table=table, ...)
EvMore(caller, pages, page_formatter=_to_evtable)
```
""" """
self._caller = caller self._caller = caller
@ -216,15 +219,17 @@ class EvMore(object):
self.exit_on_lastpage = exit_on_lastpage self.exit_on_lastpage = exit_on_lastpage
self.exit_cmd = exit_cmd self.exit_cmd = exit_cmd
self._exit_msg = "Exited |wmore|n pager." self._exit_msg = "Exited |wmore|n pager."
self._page_formatter = page_formatter
self._kwargs = kwargs self._kwargs = kwargs
self._data = None self._data = None
self._paginator = None
self._pages = [] self._pages = []
self._npages = 1
self._npos = 0 self._npos = 0
self._npages = 1
self._paginator = self.paginator_index
self._page_formatter = str
# set up individual pages for different sessions # set up individual pages for different sessions
height = max(4, session.protocol_flags.get("SCREENHEIGHT", {0: _SCREEN_HEIGHT})[0] - 4) height = max(4, session.protocol_flags.get("SCREENHEIGHT", {0: _SCREEN_HEIGHT})[0] - 4)
self.width = session.protocol_flags.get("SCREENWIDTH", {0: _SCREEN_WIDTH})[0] self.width = session.protocol_flags.get("SCREENWIDTH", {0: _SCREEN_WIDTH})[0]
@ -232,155 +237,19 @@ class EvMore(object):
self.height = min(10000 // max(1, self.width), height) self.height = min(10000 // max(1, self.width), height)
# does initial parsing of input # does initial parsing of input
self.parse_input(inp) self.init_pages(inp)
# kick things into gear # kick things into gear
self.start() self.start()
# Hooks for customizing input handling and formatting (use if overriding this class) # EvMore functional methods
def parse_input(self, inp):
"""
Parse the input to figure out the size of the data, how many pages it
consist of and pick the correct paginator mechanism. Override this if
you want to support a new type of input.
Each initializer should set self._paginator and optionally self._page_formatter
for properly handling the input data.
"""
if inherits_from(inp, "evennia.utils.evtable.EvTable"):
# an EvTable
self.init_evtable(inp)
elif isinstance(inp, QuerySet):
# a queryset
self.init_queryset(inp)
elif isinstance(inp, Paginator):
self.init_django_paginator(inp)
elif not isinstance(inp, str):
# anything else not a str
self.init_iterable(inp)
elif "\f" in inp:
# string with \f line-break markers in it
self.init_f_str(inp)
else:
# a string
self.init_str(inp)
def format_page(self, page):
"""
Page formatter. Uses the page_formatter callable by default.
This allows to easier override the class if needed.
Args:
page (any): A piece of data representing one page to display. This must
be poss
Returns:
"""
return self._page_formatter(page)
# paginators - responsible for extracting a specific page number
def paginator_index(self, pageno):
"""Paginate to specific, known index"""
return self._data[pageno]
def paginator_slice(self, pageno):
"""
Paginate by slice. This is done with an eye on memory efficiency (usually for
querysets); to avoid fetching all objects at the same time.
"""
return self._data[pageno * self.height: pageno * self.height + self.height]
def paginator_django(self, pageno):
"""
Paginate using the django queryset Paginator API. Note that his is indexed from 1.
"""
return self._data.page(pageno + 1)
# inits for different input types
def init_evtable(self, table):
"""The input is an EvTable."""
if table.height:
# enforced height of each paged table, plus space for evmore extras
self.height = table.height - 4
# convert table to string
text = str(table)
self._justify = False
self._justify_kwargs = None # enforce
self.init_str(text)
def init_queryset(self, qs):
"""The input is a queryset"""
nsize = qs.count() # we assume each will be a line
self._npages = nsize // self.height + (0 if nsize % self.height == 0 else 1)
self._data = qs
self._paginator = self.paginator_slice
def init_django_paginator(self, pages):
"""
The input is a django Paginator object.
"""
self._npages = pages.num_pages
self._data = pages
self._paginator = self.paginator_django
def init_iterable(self, inp):
"""The input is something other than a string - convert to iterable of strings"""
inp = make_iter(inp)
nsize = len(inp)
self._npages = nsize // self.height + (0 if nsize % self.height == 0 else 1)
self._data = inp
self._paginator = self.paginator_slice
def init_f_str(self, text):
"""
The input contains \f markers. We use \f to indicate the user wants to
enforce their line breaks on their own. If so, we do no automatic
line-breaking/justification at all.
"""
self._data = text.split("\f")
self._npages = len(self._data)
self._paginator = self.paginator_index
def init_str(self, text):
"""The input is a string"""
if self._justify:
# we must break very long lines into multiple ones. Note that this
# will also remove spurious whitespace.
justify_kwargs = self._justify_kwargs or {}
width = self._justify_kwargs.get("width", self.width)
justify_kwargs["width"] = width
justify_kwargs["align"] = self._justify_kwargs.get("align", "l")
justify_kwargs["indent"] = self._justify_kwargs.get("indent", 0)
lines = []
for line in text.split("\n"):
if len(line) > width:
lines.extend(justify(line, **justify_kwargs).split("\n"))
else:
lines.append(line)
else:
# no justification. Simple division by line
lines = text.split("\n")
self._data = [
"\n".join(lines[i : i + self.height]) for i in range(0, len(lines), self.height)
]
self._npages = len(self._data)
self._paginator = self.paginator_index
# display helpers and navigation
def display(self, show_footer=True): def display(self, show_footer=True):
""" """
Pretty-print the page. Pretty-print the page.
""" """
pos = self._npos pos = self._npos
text = self.format_page(self._paginator(pos)) text = self.page_formatter(self.paginator(pos))
if show_footer: if show_footer:
page = _DISPLAY.format(text=text, pageno=pos + 1, pagemax=self._npages) page = _DISPLAY.format(text=text, pageno=pos + 1, pagemax=self._npages)
else: else:
@ -460,6 +329,173 @@ class EvMore(object):
self.page_top() self.page_top()
# default paginators - responsible for extracting a specific page number
def paginator_index(self, pageno):
"""Paginate to specific, known index"""
return self._data[pageno]
def paginator_slice(self, pageno):
"""
Paginate by slice. This is done with an eye on memory efficiency (usually for
querysets); to avoid fetching all objects at the same time.
"""
return self._data[pageno * self.height: pageno * self.height + self.height]
def paginator_django(self, pageno):
"""
Paginate using the django queryset Paginator API. Note that his is indexed from 1.
"""
return self._data.page(pageno + 1)
# default helpers to set up particular input types
def init_evtable(self, table):
"""The input is an EvTable."""
if table.height:
# enforced height of each paged table, plus space for evmore extras
self.height = table.height - 4
# convert table to string
text = str(table)
self._justify = False
self._justify_kwargs = None # enforce
self.init_str(text)
def init_queryset(self, qs):
"""The input is a queryset"""
nsize = qs.count() # we assume each will be a line
self._npages = nsize // self.height + (0 if nsize % self.height == 0 else 1)
self._data = qs
def init_django_paginator(self, pages):
"""
The input is a django Paginator object.
"""
self._npages = pages.num_pages
self._data = pages
def init_iterable(self, inp):
"""The input is something other than a string - convert to iterable of strings"""
inp = make_iter(inp)
nsize = len(inp)
self._npages = nsize // self.height + (0 if nsize % self.height == 0 else 1)
self._data = inp
def init_f_str(self, text):
"""
The input contains \f markers. We use \f to indicate the user wants to
enforce their line breaks on their own. If so, we do no automatic
line-breaking/justification at all.
"""
self._data = text.split("\f")
self._npages = len(self._data)
def init_str(self, text):
"""The input is a string"""
if self._justify:
# we must break very long lines into multiple ones. Note that this
# will also remove spurious whitespace.
justify_kwargs = self._justify_kwargs or {}
width = self._justify_kwargs.get("width", self.width)
justify_kwargs["width"] = width
justify_kwargs["align"] = self._justify_kwargs.get("align", "l")
justify_kwargs["indent"] = self._justify_kwargs.get("indent", 0)
lines = []
for line in text.split("\n"):
if len(line) > width:
lines.extend(justify(line, **justify_kwargs).split("\n"))
else:
lines.append(line)
else:
# no justification. Simple division by line
lines = text.split("\n")
self._data = [
"\n".join(lines[i : i + self.height]) for i in range(0, len(lines), self.height)
]
self._npages = len(self._data)
# Hooks for customizing input handling and formatting (override in a child class)
def init_pages(self, inp):
"""
Initialize the pagination. By default, will analyze input type to determine
how pagination automatically.
Args:
inp (any): Incoming data to be paginated. By default, handles pagination of
strings, querysets, django.Paginator, EvTables and any iterables with strings.
Notes:
If overridden, this method must perform the following actions:
- read and re-store `self._data` (the incoming data set) if needed for pagination to work.
- set `self._npages` to the total number of pages. Default is 1.
- set `self._paginator` to a callable that will take a page number 1...N and return
the data to display on that page (not any decorations or next/prev buttons). If only
wanting to change the paginator, override `self.paginator` instead.
- set `self._page_formatter` to a callable that will receive the page from `self._paginator`
and format it with one element per line. Default is `str`. Or override `self.page_formatter`
directly instead.
By default, helper methods are called that perform these actions
depending on supported inputs.
"""
if inherits_from(inp, "evennia.utils.evtable.EvTable"):
# an EvTable
self.init_evtable(inp)
elif isinstance(inp, QuerySet):
# a queryset
self.init_queryset(inp)
self._paginator = self.paginator_slice
elif isinstance(inp, Paginator):
self.init_django_paginator(inp)
self._paginator = self.paginator_django
elif not isinstance(inp, str):
# anything else not a str
self.init_iterable(inp)
self._paginator = self.paginator_django
elif "\f" in inp:
# string with \f line-break markers in it
self.init_f_str(inp)
else:
# a string
self.init_str(inp)
def paginator(self, pageno):
"""
Paginator. The data operated upon is in `self._data`.
Args:
pageno (int): The page number to view, from 1...N
Returns:
str: The page to display (without any decorations, those are added
by EvMore).
"""
return self._paginator(pageno)
def page_formatter(self, page):
"""
Page formatter. Every page passes through this method. Override
it to customize behvaior per-page. A common use is to generate a new
EvTable for every page (this is more efficient than to generate one huge
EvTable across many pages and feed it into EvMore all at once).
Args:
page (any): A piece of data representing one page to display. This must
Returns:
str: A ready-formatted page to display. Extra footer with help about
switching to the next/prev page will be added automatically
"""
return self._page_formatter(page)
# helper function # helper function