Gave a more informative error message when reading non-UTF-8 batchfiles containing international symbols (issue97) as well as refactored the processors a bit further. Fixed some minor typographic details in some other commands.

game/gamesrc/commands/default/batchprocess.py

@@ -29,8 +29,24 @@ from src.commands.cmdset import CmdSet
 
 
 HEADER_WIDTH = 70
+UTF8_ERROR = \
+"""
+ {rDecode error in '%s'.{n
+
+ This file contains non-ascii character(s). This is common if you
+ wrote some input in a language that has more letters and special
+ symbols than English; such as accents or umlauts.  This is usually
+ fine and fully supported! But for Evennia to know how to decode such
+ characters in a universal way, the batchfile must be saved with the
+ international 'UTF-8' encoding. This file is not.
+ 
+ Please re-save the batchfile with the UTF-8 encoding (refer to the
+ documentation of your text editor on how to do this, or switch to a 
+ better featured one) and try again. 
+
+ The (first) error was found with a character on line %s in the file.
+"""
 
-#global defines for storage
 
 #------------------------------------------------------------
 # Helper functions
@@ -194,7 +210,13 @@ class CmdBatchCommands(MuxCommand):
 
         #parse indata file        
 
-        commands = BATCHCMD.parse_file(python_path)
+        try:
+            commands = BATCHCMD.parse_file(python_path)
+        except UnicodeDecodeError, err:
+            lnum = err.linenum
+            caller.msg(UTF8_ERROR % (python_path, lnum))
+            return 
+
         if not commands:
             string = "'%s' not found.\nYou have to supply the python path "
             string += "of the file relative to \nyour batch-file directory (%s)."
@@ -271,7 +293,13 @@ class CmdBatchCode(MuxCommand):
         python_path = self.args
 
         #parse indata file
-        codes = BATCHCODE.parse_file(python_path)
+        try:
+            codes = BATCHCODE.parse_file(python_path)
+        except UnicodeDecodeError, err:
+            lnum = err.linenum
+            caller.msg(UTF8_ERROR % (python_path, lnum))
+            return 
+
         if not codes:
             string = "'%s' not found.\nYou have to supply the python path "
             string += "of the file relative to \nyour batch-file directory (%s)."

game/gamesrc/commands/default/general.py

@@ -491,11 +491,11 @@ class CmdSay(MuxCommand):
         speech = caller.location.at_say(caller, speech)
 
         # Feedback for the object doing the talking.
-        caller.msg("You say, '%s'" % speech)
+        caller.msg('You say, "%s{n"' % speech)
         
         # Build the string to emit to neighbors.
-        emit_string = "{c%s{n says, '%s'" % (caller.name, 
+        emit_string = '{c%s{n says, "%s{n"' % (caller.name, 
-                                             speech)
+                                               speech)
         caller.location.msg_contents(emit_string, 
                                          exclude=caller)
 

game/gamesrc/commands/default/objmanip.py

@@ -205,7 +205,6 @@ class CmdSetObjAlias(MuxCommand):
         obj.aliases = aliases 
         caller.msg("Aliases for '%s' are now set to %s." % (obj.name, aliases))
 
-
 class CmdName(ObjManipCommand):
     """
     cname - change the name and/or aliases of an object

game/gamesrc/world/examples/batch_cmds.ev

@@ -48,7 +48,7 @@ know you want to!
 # Now let's place the button where it belongs (let's say limbo #2 is 
 # the evil lair in our example)
 
-@teleport #2
+teleport #2
 
 #... and drop it (remember, this comment ends input to @teleport, so don't
 #forget it!) The very last command in the file need not be ended with #.

src/utils/batchprocessors.py

@@ -96,9 +96,9 @@ Code blocks are separated by python comments starting with special code words.
 
 #HEADER - this denotes commands global to the entire file, such as
           import statements and global variables. They will
-          automatically be made available for each block.  Observe
+          automatically be pasted at the top of all code blocks. Observe
           that changes to these variables made in one block is not
-          preserved between blocks!)
+          preserved between blocks!
 #CODE [objname, objname, ...] - This designates a code block that will be executed like a 
            stand-alone piece of code together with any #HEADER
            defined. <objname>s mark the (variable-)names of objects created in the code, 
@@ -138,16 +138,71 @@ script = create.create_script()
 """
 
 import re
+import codecs
+from traceback import format_exc
 from django.conf import settings
+from django.core.management import setup_environ
 from src.utils import logger
 from src.utils import utils
-#from src.commands.cmdset import CmdSet
-#from src.scripts.scripts import Script
 from game import settings as settings_module
-from django.core.management import setup_environ
-from traceback import format_exc
 
 
+#------------------------------------------------------------
+# Helper function
+#------------------------------------------------------------
+
+def read_batchfile(pythonpath, file_ending='.py', file_encoding='utf-8'):
+    """
+    This reads the contents of a batch-file.
+    Filename is considered to be the name of the batch file
+    relative the directory specified in settings.py.
+
+    file_ending specify which batchfile ending should be 
+    assumed (.ev or .py).
+    """    
+
+    # open the file
+
+    if pythonpath and not (pythonpath.startswith('src.') or 
+                           pythonpath.startswith('game.')):
+        pythonpath = "%s.%s" % (settings.BASE_BATCHPROCESS_PATH, 
+                                pythonpath)
+    abspath = utils.pypath_to_realpath(pythonpath, file_ending)
+    try:        
+        # we read the file directly into unicode.
+        fobj = codecs.open(abspath, 'r', encoding=file_encoding)
+    except IOError:
+        # try again without the appended file ending
+        abspath2 = utils.pypath_to_realpath(pythonpath, None)
+        try:
+            fobj = codecs.open(abspath, 'r', encoding=file_encoding)
+        except IOError:            
+            string = "Could not open batchfile '%s', nor '%s'."
+            logger.log_errmsg(string % (abspath2, abspath))
+        return None
+    
+    # We have successfully found and opened the file. Now actually
+    # try to decode it using the given protocol. 
+
+    try:
+        lines = fobj.readlines()
+    except UnicodeDecodeError:
+        # give the line of failure
+        fobj.seek(0)
+        try:
+            lnum = 0
+            for lnum, line in enumerate(fobj):
+                pass
+        except UnicodeDecodeError, err:
+            # lnum starts from 0, so we add +1 line, 
+            # besides the faulty line is never read
+            # so we add another 1 (thus +2) to get
+            # the actual line number seen in an editor. 
+            err.linenum = lnum + 2
+            raise err
+    fobj.close()
+    return lines
+
 #------------------------------------------------------------
 #
 # Batch-command processor
@@ -159,28 +214,6 @@ class BatchCommandProcessor(object):
     This class implements a batch-command processor.
 
     """   
-    
-    def read_file(self, pythonpath):
-        """
-        This reads the contents of a batch-command file.
-        Filename is considered to be the name of the batch file
-        relative the directory specified in settings.py
-        """    
-
-        if pythonpath and not (pythonpath.startswith('src.') or 
-                               pythonpath.startswith('game.')):
-            pythonpath = "%s.%s" % (settings.BASE_BATCHPROCESS_PATH, 
-                                    pythonpath)
-        abspath = utils.pypath_to_realpath(pythonpath, 'ev')
-        try:
-            fobj = open(abspath)
-        except IOError:
-            logger.log_errmsg("Could not open path '%s'." % abspath)
-            return None
-        lines = fobj.readlines()
-        fobj.close()
-        return lines
-
     def parse_file(self, pythonpath):
         """
         This parses the lines of a batchfile according to the following
@@ -212,7 +245,9 @@ class BatchCommandProcessor(object):
                 return "empty"
 
         #read the indata, if possible.
-        lines = self.read_file(pythonpath)
+        lines = read_batchfile(pythonpath, file_ending='.ev')
+        
+        #line = utils.to_unicode(line)
         if not lines:
             return None 
 
@@ -225,6 +260,7 @@ class BatchCommandProcessor(object):
 
         #parse all command definitions into a list.
         for line in lines:
+                        
             typ = identify_line(line)
             if typ == "commanddef":
                 curr_cmd += line
@@ -258,28 +294,6 @@ class BatchCodeProcessor(object):
  
     """
 
-    def read_file(self, pythonpath):
-        """
-        This reads the contents of batchfile.
-        Filename is considered to be the name of the batch file
-        relative the directory specified in settings.py
-        """    
-
-        if pythonpath and not (pythonpath.startswith('src.') or 
-                               pythonpath.startswith('game.')):
-            pythonpath = "%s.%s" % (settings.BASE_BATCHPROCESS_PATH, 
-                                    pythonpath)
-        abspath = utils.pypath_to_realpath(pythonpath, 'py')
-        try:
-            fobj = open(abspath)
-        except IOError:
-            logger.log_errmsg("Could not open path '%s'." % abspath)
-            return None
-        lines = fobj.readlines()
-        fobj.close()
-        return lines
-
-
     def parse_file(self, pythonpath):
         """
         This parses the lines of a batchfile according to the following
@@ -325,7 +339,7 @@ class BatchCodeProcessor(object):
 
         # read indata
 
-        lines = self.read_file(pythonpath)
+        lines = read_batchfile(pythonpath, file_ending='.py')
         if not lines:
             return None
 

src/utils/utils.py

@@ -149,11 +149,11 @@ def get_evennia_version():
     except IOError:
         return "Unknown version"
         
-def pypath_to_realpath(python_path, file_ending='py'):
+def pypath_to_realpath(python_path, file_ending='.py'):
     """
-    Converts a path on dot python form (e.g. src.objects.models)
+    Converts a path on dot python form (e.g. 'src.objects.models') to
-    to a system path (src/objects/models.py). Calculates all
+    a system path (src/objects/models.py). Calculates all paths as
-    paths starting from the evennia main directory.
+    absoulte paths starting from the evennia main directory.
     """
     pathsplit = python_path.strip().split('.')
     if not pathsplit:
@@ -161,14 +161,15 @@ def pypath_to_realpath(python_path, file_ending='py'):
     path = settings.BASE_PATH 
     for directory in pathsplit:
         path = os.path.join(path, directory)
-    return "%s.%s" % (path, file_ending) 
+    if file_ending:
+        return "%s%s" % (path, file_ending) 
+    return path 
 
 def dbref(dbref):
     """
-    Converts/checks if input is a valid dbref
+    Converts/checks if input is a valid dbref Valid forms of dbref
-    Valid forms of dbref (database reference number)
+    (database reference number) are either a string '#N' or 
-    are either a string '#N' or an integer N.
+    an integer N.  Output is the integer part.
-    Output is the integer part. 
     """
     if type(dbref) == str:
         dbref = dbref.lstrip('#')