From 7285cab2db6853e03d61c589a1a82b765e14c686 Mon Sep 17 00:00:00 2001 From: Griatch Date: Wed, 24 Aug 2016 23:44:29 +0200 Subject: [PATCH] Refactor batchcode processor to be more stable. Did multiple clean-ups. Implements #939. Fixes #909. Fixes #937. --- evennia/commands/default/batchprocess.py | 51 +++-- .../tutorial_examples/example_batch_code.py | 11 +- evennia/utils/batchprocessors.py | 194 ++++++++---------- 3 files changed, 124 insertions(+), 132 deletions(-) diff --git a/evennia/commands/default/batchprocess.py b/evennia/commands/default/batchprocess.py index 22b9d9333..d9aa7f223 100644 --- a/evennia/commands/default/batchprocess.py +++ b/evennia/commands/default/batchprocess.py @@ -17,6 +17,7 @@ the Evennia API. It is also a severe security risk and should therefore always be limited to superusers only. """ +import re from builtins import range from django.conf import settings @@ -24,8 +25,11 @@ from evennia.utils.batchprocessors import BATCHCMD, BATCHCODE from evennia.commands.cmdset import CmdSet from evennia.utils import logger, utils -COMMAND_DEFAULT_CLASS = utils.class_from_module(settings.COMMAND_DEFAULT_CLASS) -#from evennia.commands.default.muxcommand import COMMAND_DEFAULT_CLASS + +_RE_COMMENT = re.compile(r"^#.*?$", re.MULTILINE + re.DOTALL) +_RE_CODE_START = re.compile(r"^# batchcode code:", re.MULTILINE) +_COMMAND_DEFAULT_CLASS = utils.class_from_module(settings.COMMAND_DEFAULT_CLASS) +#from evennia.commands.default.muxcommand import _COMMAND_DEFAULT_CLASS # limit symbols for API inclusion __all__ = ("CmdBatchCommands", "CmdBatchCode") @@ -88,7 +92,10 @@ def format_header(caller, entry): Formats a header """ width = _HEADER_WIDTH - 10 - entry = entry.strip() + # strip all comments for the header + entry = _RE_CODE_START.split(entry, 1)[1] + entry = _RE_COMMENT.sub("", entry).strip() + header = utils.crop(entry, width=width) ptr = caller.ndb.batch_stackptr + 1 stacklen = len(caller.ndb.batch_stack) @@ -207,7 +214,7 @@ def purge_processor(caller): #------------------------------------------------------------ -class CmdBatchCommands(COMMAND_DEFAULT_CLASS): +class CmdBatchCommands(_COMMAND_DEFAULT_CLASS): """ build from batch-command file @@ -310,7 +317,7 @@ class CmdBatchCommands(COMMAND_DEFAULT_CLASS): purge_processor(caller) -class CmdBatchCode(COMMAND_DEFAULT_CLASS): +class CmdBatchCode(_COMMAND_DEFAULT_CLASS): """ build from batch-code file @@ -348,7 +355,7 @@ class CmdBatchCode(COMMAND_DEFAULT_CLASS): #parse indata file try: - codes = BATCHCODE.parse_file(python_path, debug=debug) + codes = BATCHCODE.parse_file(python_path) except UnicodeDecodeError as err: caller.msg(_UTF8_ERROR % (python_path, err)) return @@ -421,7 +428,7 @@ class CmdBatchCode(COMMAND_DEFAULT_CLASS): # (these are the same for both processors) #------------------------------------------------------------ -class CmdStateAbort(COMMAND_DEFAULT_CLASS): +class CmdStateAbort(_COMMAND_DEFAULT_CLASS): """ @abort @@ -439,7 +446,7 @@ class CmdStateAbort(COMMAND_DEFAULT_CLASS): self.caller.msg("Exited processor and reset out active cmdset back to the default one.") -class CmdStateLL(COMMAND_DEFAULT_CLASS): +class CmdStateLL(_COMMAND_DEFAULT_CLASS): """ ll @@ -453,7 +460,7 @@ class CmdStateLL(COMMAND_DEFAULT_CLASS): def func(self): show_curr(self.caller, showall=True) -class CmdStatePP(COMMAND_DEFAULT_CLASS): +class CmdStatePP(_COMMAND_DEFAULT_CLASS): """ pp @@ -474,7 +481,7 @@ class CmdStatePP(COMMAND_DEFAULT_CLASS): batch_cmd_exec(caller) -class CmdStateRR(COMMAND_DEFAULT_CLASS): +class CmdStateRR(_COMMAND_DEFAULT_CLASS): """ rr @@ -496,7 +503,7 @@ class CmdStateRR(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateRRR(COMMAND_DEFAULT_CLASS): +class CmdStateRRR(_COMMAND_DEFAULT_CLASS): """ rrr @@ -518,7 +525,7 @@ class CmdStateRRR(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateNN(COMMAND_DEFAULT_CLASS): +class CmdStateNN(_COMMAND_DEFAULT_CLASS): """ nn @@ -539,7 +546,7 @@ class CmdStateNN(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateNL(COMMAND_DEFAULT_CLASS): +class CmdStateNL(_COMMAND_DEFAULT_CLASS): """ nl @@ -561,7 +568,7 @@ class CmdStateNL(COMMAND_DEFAULT_CLASS): show_curr(caller, showall=True) -class CmdStateBB(COMMAND_DEFAULT_CLASS): +class CmdStateBB(_COMMAND_DEFAULT_CLASS): """ bb @@ -583,7 +590,7 @@ class CmdStateBB(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateBL(COMMAND_DEFAULT_CLASS): +class CmdStateBL(_COMMAND_DEFAULT_CLASS): """ bl @@ -605,7 +612,7 @@ class CmdStateBL(COMMAND_DEFAULT_CLASS): show_curr(caller, showall=True) -class CmdStateSS(COMMAND_DEFAULT_CLASS): +class CmdStateSS(_COMMAND_DEFAULT_CLASS): """ ss [steps] @@ -634,7 +641,7 @@ class CmdStateSS(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateSL(COMMAND_DEFAULT_CLASS): +class CmdStateSL(_COMMAND_DEFAULT_CLASS): """ sl [steps] @@ -663,7 +670,7 @@ class CmdStateSL(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateCC(COMMAND_DEFAULT_CLASS): +class CmdStateCC(_COMMAND_DEFAULT_CLASS): """ cc @@ -695,7 +702,7 @@ class CmdStateCC(COMMAND_DEFAULT_CLASS): caller.msg(format_code("Finished processing batch file.")) -class CmdStateJJ(COMMAND_DEFAULT_CLASS): +class CmdStateJJ(_COMMAND_DEFAULT_CLASS): """ jj @@ -719,7 +726,7 @@ class CmdStateJJ(COMMAND_DEFAULT_CLASS): show_curr(caller) -class CmdStateJL(COMMAND_DEFAULT_CLASS): +class CmdStateJL(_COMMAND_DEFAULT_CLASS): """ jl @@ -743,7 +750,7 @@ class CmdStateJL(COMMAND_DEFAULT_CLASS): show_curr(caller, showall=True) -class CmdStateQQ(COMMAND_DEFAULT_CLASS): +class CmdStateQQ(_COMMAND_DEFAULT_CLASS): """ qq @@ -758,7 +765,7 @@ class CmdStateQQ(COMMAND_DEFAULT_CLASS): self.caller.msg("Aborted interactive batch mode.") -class CmdStateHH(COMMAND_DEFAULT_CLASS): +class CmdStateHH(_COMMAND_DEFAULT_CLASS): "Help command" key = "hh" diff --git a/evennia/contrib/tutorial_examples/example_batch_code.py b/evennia/contrib/tutorial_examples/example_batch_code.py index 7e0d8586e..a38be7b12 100644 --- a/evennia/contrib/tutorial_examples/example_batch_code.py +++ b/evennia/contrib/tutorial_examples/example_batch_code.py @@ -51,7 +51,7 @@ from evennia import DefaultObject limbo = search_object('Limbo')[0] -#CODE (create red button) +#CODE # This is the first code block. Within each block, Python # code works as normal. Note how we make use if imports and @@ -66,7 +66,7 @@ red_button = create_object(red_button.RedButton, key="Red button", # we take a look at what we created caller.msg("A %s was created." % red_button.key) -#CODE (create table and chair) table, chair +#CODE # this code block has 'table' and 'chair' set as deletable # objects. This means that when the batchcode processor runs in @@ -80,5 +80,10 @@ caller.msg("A %s was created." % red_button.key) table = create_object(DefaultObject, key="Table", location=limbo) chair = create_object(DefaultObject, key="Chair", location=limbo) -string = "A %s and %s were created. If debug was active, they were deleted again." +string = "A %s and %s were created." +if DEBUG: + string += " Since debug was active, they were deleted again." + table.delete() + chair.delete() + caller.msg(string % (table, chair)) diff --git a/evennia/utils/batchprocessors.py b/evennia/utils/batchprocessors.py index 8b721d0e8..65c9bb5d7 100644 --- a/evennia/utils/batchprocessors.py +++ b/evennia/utils/batchprocessors.py @@ -101,43 +101,43 @@ An example batch file is `contrib/examples/batch_example.ev`. Batch-code processor file syntax The Batch-code processor accepts full python modules (e.g. `batch.py`) -that looks identical to normal Python files with a few exceptions that -allows them to the executed in blocks. This way of working assures a -sequential execution of the file and allows for features like stepping -from block to block (without executing those coming before), as well -as automatic deletion of created objects etc. You can however also run -a batch-code Python file directly using Python. +that looks identical to normal Python files. The difference from +importing and running any Python module is that the batch-code module +is loaded as a file and executed directly, so changes to the file will +apply immediately without a server @reload. -Code blocks are separated by python comments starting with special -code words: +Optionally, one can add some special commented tokens to split the +execution of the code for the benefit of the batchprocessor's +interactive- and debug-modes. This allows to conveniently step through +the code and re-run sections of it easily during development. -HEADER - this denotes commands global to the entire file, such as - import statements and global variables. They will - automatically be pasted at the top of all code - blocks. Observe that changes to these variables made in one - block is not preserved between blocks! -CODE -CODE (info) -CODE (info) objname1, objname1, ... - - This designates a code block that will be executed like a - stand-alone piece of code together with any #HEADER - defined. (info) text is used by the interactive mode to - display info about the node to run. s mark the - (variable-)names of objects created in the code, and which - may be auto-deleted by the processor if desired (such as - when debugging the script). E.g., if the code contains the - command myobj = create.create_object(...), you could put - 'myobj' in the #CODE header regardless of what the created - object is actually called in-game. -INSERT path.filename - This imports another batch_code.py file and - runs it in the given position. paths are given as python - path. The inserted file will retain its own HEADERs which - will not be mixed with the HEADERs of the file importing - this file. +Code blocks are marked by commented tokens alone on a line: -The following variables are automatically made available for the script: +#HEADER - This denotes code that should be pasted at the top of all + other code. Multiple HEADER statements - regardless of where + it exists in the file - is the same as one big block. + Observe that changes to variables made in one block is not + preserved between blocks! +#CODE - This designates a code block that will be executed like a + stand-alone piece of code together with any HEADER(s) + defined. It is mainly used as a way to mark stop points for + the interactive mode of the batchprocessor. If no CODE block + is defined in the module, the entire module (including HEADERS) + is assumed to be a CODE block. +#INSERT path.filename - This imports another batch_code.py file and + runs it in the given position. The inserted file will retain + its own HEADERs which will not be mixed with the headers of + this file. + +Importing works as normal. The following variables are automatically +made available in the script namespace. + +- `caller` - The object executing the batchscript +- `DEBUG` - This is a boolean marking if the batchprocessor is running + in debug mode. It can be checked to e.g. delete created objects + when running a CODE block multiple times during testing. + (avoids creating a slew of same-named db objects) -caller - the object executing the script Example batch.py file ----------------------------------- @@ -151,7 +151,7 @@ from types import basetypes GOLD = 10 -#CODE obj, obj2 +#CODE obj = create.create_object(basetypes.Object) obj2 = create.create_object(basetypes.Object) @@ -159,6 +159,10 @@ obj.location = caller.location obj.db.gold = GOLD caller.msg("The object was created!") +if DEBUG: + obj.delete() + obj2.delete() + #INSERT another_batch_file #CODE @@ -175,13 +179,11 @@ import sys from django.conf import settings from evennia.utils import utils -ENCODINGS = settings.ENCODINGS -CODE_INFO_HEADER = re.compile(r"\(.*?\)") - -RE_INSERT = re.compile(r"^\#INSERT (.*)", re.MULTILINE) -RE_CLEANBLOCK = re.compile(r"^\#.*?$|^\s*$", re.MULTILINE) -RE_CMD_SPLIT = re.compile(r"^\#.*?$", re.MULTILINE) -RE_CODE_SPLIT = re.compile(r"(^\#CODE.*?$|^\#HEADER)$", re.MULTILINE) +_ENCODINGS = settings.ENCODINGS +_RE_INSERT = re.compile(r"^\#INSERT (.*)", re.MULTILINE) +_RE_CLEANBLOCK = re.compile(r"^\#.*?$|^\s*$", re.MULTILINE) +_RE_CMD_SPLIT = re.compile(r"^\#.*?$", re.MULTILINE) +_RE_CODE_OR_HEADER = re.compile(r"(\A|^\#CODE|^\#HEADER).*?$(.*?)(?=^#CODE.*?$|^#HEADER.*?$|\Z)", re.MULTILINE + re.DOTALL) #------------------------------------------------------------ @@ -219,7 +221,7 @@ def read_batchfile(pythonpath, file_ending='.py'): for abspath in abspaths: # try different paths, until we get a match # we read the file directly into unicode. - for file_encoding in ENCODINGS: + for file_encoding in _ENCODINGS: # try different encodings, in order try: with codecs.open(abspath, 'r', encoding=file_encoding) as fobj: @@ -273,10 +275,10 @@ class BatchCommandProcessor(object): return "\n#".join(self.parse_file(match.group(1))) # insert commands from inserted files - text = RE_INSERT.sub(replace_insert, text) + text = _RE_INSERT.sub(replace_insert, text) #text = re.sub(r"^\#INSERT (.*?)", replace_insert, text, flags=re.MULTILINE) # get all commands - commands = RE_CMD_SPLIT.split(text) + commands = _RE_CMD_SPLIT.split(text) #commands = re.split(r"^\#.*?$", text, flags=re.MULTILINE) #remove eventual newline at the end of commands commands = [c.strip('\r\n') for c in commands] @@ -309,79 +311,58 @@ class BatchCodeProcessor(object): """ - def parse_file(self, pythonpath, debug=False): + def parse_file(self, pythonpath): """ This parses the lines of a batchfile according to the following rules: Args: pythonpath (str): The dot-python path to the file. - debug (bool, optional): Insert delete-commands for - deleting created objects. Returns: - codeblocks (list): A list of all #CODE blocks. + codeblocks (list): A list of all #CODE blocks, each with + prepended #HEADER data. If no #CODE blocks were found, + this will be a list of one element. Notes: - 1. Lines starting with #HEADER starts a header block (ends other blocks) - 2. Lines starting with #CODE begins a code block (ends other blocks) - 3. #CODE headers may be of the following form: - #CODE (info) objname, objname2, ... - 4. Lines starting with #INSERT are on form #INSERT filename. - 5. All lines outside blocks are stripped. - 6. All excess whitespace beginning/ending a block is stripped. + 1. Code before a #CODE/HEADER block are considered part of + the first code/header block or is the ONLY block if no + #CODE/HEADER blocks are defined. + 2. Lines starting with #HEADER starts a header block (ends other blocks) + 3. Lines starting with #CODE begins a code block (ends other blocks) + 4. Lines starting with #INSERT are on form #INSERT filename. Code from + this file are processed with their headers *separately* before + being inserted at the point of the #INSERT. + 5. Code after the last block is considered part of the last header/code + block """ text = "".join(read_batchfile(pythonpath, file_ending='.py')) - def clean_block(text): - text = RE_CLEANBLOCK.sub("", text) - #text = re.sub(r"^\#.*?$|^\s*$", "", text, flags=re.MULTILINE) - return "\n".join([line for line in text.split("\n") if line]) - def replace_insert(match): - "Map replace entries" - return "\#\n".join(self.parse_file(match.group(1))) + "Run parse_file on the import before sub:ing it into this file" + path = match.group(1) + return "# batchcode insert (%s):" % path + "\n".join(self.parse_file(path)) + + # process and then insert code from all #INSERTS + text = _RE_INSERT.sub(replace_insert, text) - text = RE_INSERT.sub(replace_insert, text) - #text = re.sub(r"^\#INSERT (.*?)", replace_insert, text, flags=re.MULTILINE) - blocks = RE_CODE_SPLIT.split(text) - #blocks = re.split(r"(^\#CODE.*?$|^\#HEADER)$", text, flags=re.MULTILINE) headers = [] - codes = [] # list of tuples (code, info, objtuple) - if blocks: - if blocks[0]: - # the first block is either empty or an unmarked code block - code = clean_block(blocks.pop(0)) - if code: - codes.append((code, "")) - iblock = 0 - for block in blocks[::2]: - # loop over every second component; these are the #CODE/#HEADERs - if block.startswith("#HEADER"): - headers.append(clean_block(blocks[iblock + 1])) - elif block.startswith("#CODE"): - match = re.search(r"\(.*?\)", block) - info = match.group() if match else "" - objs = [] - if debug: - # insert auto-delete lines into code - objs = block[match.end():].split(",") - objs = ["# added by Evennia's debug mode\n%s.delete()" % obj.strip() for obj in objs if obj] - # build the code block - code = "\n".join([clean_block(blocks[iblock + 1])] + objs) - if code: - codes.append((code, info)) - iblock += 2 + codes = [] + for imatch, match in enumerate(list(_RE_CODE_OR_HEADER.finditer(text))): + type = match.group(1) + istart, iend = match.span(2) + code = text[istart:iend] + if type == "#HEADER": + headers.append(code) + else: # either #CODE or matching from start of file + codes.append(code) - # join the headers together to one header - headers = "\n".join(headers) - if codes: - # add the headers at the top of each non-empty block - codes = ["%s\n%s\n%s" % ("#CODE %s: " % tup[1], headers, tup[0]) for tup in codes if tup[0]] - else: - codes = ["#CODE: \n" + headers] + # join all headers together to one + header = "# batchcode header:\n%s\n\n" % "\n\n".join(headers) if headers else "" + # add header to each code block + codes = ["%s# batchcode code:\n%s" % (header, code) for code in codes] return codes @@ -393,24 +374,23 @@ class BatchCodeProcessor(object): Args: code (str): Code to run. extra_environ (dict): Environment variables to run with code. - debug (bool, optional): Insert delete statements for objects. + debug (bool, optional): Set the DEBUG variable in the execution + namespace. Returns: err (str or None): An error code or None (ok). """ # define the execution environment - environdict = {"settings_module": settings} - environ = "settings_module.configure()" - if extra_environ: - for key, value in extra_environ.items(): - environdict[key] = value + environdict = {"settings_module": settings, "DEBUG": debug} + for key, value in extra_environ.items(): + environdict[key] = value # initializing the django settings at the top of code - code = "# auto-added by Evennia\n" \ - "try: %s\n" \ + code = "# batchcode evennia initialization: \n" \ + "try: settings_module.configure()\n" \ "except RuntimeError: pass\n" \ - "finally: del settings_module\n%s" % (environ, code) + "finally: del settings_module\n\n%s" % code # execute the block try: