Reshuffling the Evennia package into the new template paradigm.

lib/utils/__init__.py

@@ -0,0 +1,5 @@
+# simple check to determine if we are currently running under pypy.
+try:
+    import __pypy__ as is_pypy
+except ImportError:
+    is_pypy = False

lib/utils/ansi.py

@@ -0,0 +1,934 @@
+"""
+ANSI - Gives colour to text.
+
+Use the codes defined in ANSIPARSER in your text
+to apply colour to text according to the ANSI standard.
+
+Examples:
+ This is %crRed text%cn and this is normal again.
+ This is {rRed text{n and this is normal again.
+
+Mostly you should not need to call parse_ansi() explicitly;
+it is run by Evennia just before returning data to/from the
+user.
+
+"""
+import re
+from src.utils import utils
+from src.utils.utils import to_str, to_unicode
+
+# ANSI definitions
+
+ANSI_BEEP = "\07"
+ANSI_ESCAPE = "\033"
+ANSI_NORMAL = "\033[0m"
+
+ANSI_UNDERLINE = "\033[4m"
+ANSI_HILITE = "\033[1m"
+ANSI_BLINK = "\033[5m"
+ANSI_INVERSE = "\033[7m"
+ANSI_INV_HILITE = "\033[1;7m"
+ANSI_INV_BLINK = "\033[7;5m"
+ANSI_BLINK_HILITE = "\033[1;5m"
+ANSI_INV_BLINK_HILITE = "\033[1;5;7m"
+
+# Foreground colors
+ANSI_BLACK = "\033[30m"
+ANSI_RED = "\033[31m"
+ANSI_GREEN = "\033[32m"
+ANSI_YELLOW = "\033[33m"
+ANSI_BLUE = "\033[34m"
+ANSI_MAGENTA = "\033[35m"
+ANSI_CYAN = "\033[36m"
+ANSI_WHITE = "\033[37m"
+
+# Background colors
+ANSI_BACK_BLACK = "\033[40m"
+ANSI_BACK_RED = "\033[41m"
+ANSI_BACK_GREEN = "\033[42m"
+ANSI_BACK_YELLOW = "\033[43m"
+ANSI_BACK_BLUE = "\033[44m"
+ANSI_BACK_MAGENTA = "\033[45m"
+ANSI_BACK_CYAN = "\033[46m"
+ANSI_BACK_WHITE = "\033[47m"
+
+# Formatting Characters
+ANSI_RETURN = "\r\n"
+ANSI_TAB = "\t"
+ANSI_SPACE = " "
+
+# Escapes
+ANSI_ESCAPES = ("{{", "%%", "\\\\")
+
+from collections import OrderedDict
+_PARSE_CACHE = OrderedDict()
+_PARSE_CACHE_SIZE = 10000
+
+
+class ANSIParser(object):
+    """
+    A class that parses ansi markup
+    to ANSI command sequences
+
+    We also allow to escape colour codes
+    by prepending with a \ for mux-style and xterm256,
+    an extra { for Merc-style codes
+    """
+
+    def sub_ansi(self, ansimatch):
+        """
+        Replacer used by re.sub to replace ansi
+        markers with correct ansi sequences
+        """
+        return self.ansi_map.get(ansimatch.group(), "")
+
+    def sub_xterm256(self, rgbmatch):
+        """
+        This is a replacer method called by re.sub with the matched
+        tag. It must return the correct ansi sequence.
+
+        It checks self.do_xterm256 to determine if conversion
+        to standard ansi should be done or not.
+        """
+        if not rgbmatch:
+            return ""
+
+        # get tag, stripping the initial marker
+        rgbtag = rgbmatch.group()[1:]
+
+        background = rgbtag[0] == '['
+        if background:
+            red, green, blue = int(rgbtag[1]), int(rgbtag[2]), int(rgbtag[3])
+        else:
+            red, green, blue = int(rgbtag[0]), int(rgbtag[1]), int(rgbtag[2])
+
+        if self.do_xterm256:
+            colval = 16 + (red * 36) + (green * 6) + blue
+            #print "RGB colours:", red, green, blue
+            return "\033[%s8;5;%s%s%sm" % (3 + int(background), colval/100, (colval % 100)/10, colval%10)
+        else:
+            #print "ANSI convert:", red, green, blue
+            # xterm256 not supported, convert the rgb value to ansi instead
+            if red == green and red == blue and red < 2:
+                if background:
+                    return ANSI_BACK_BLACK
+                elif red >= 1:
+                    return ANSI_HILITE + ANSI_BLACK
+                else:
+                    return ANSI_NORMAL + ANSI_BLACK
+            elif red == green and red == blue:
+                if background:
+                    return ANSI_BACK_WHITE
+                elif red >= 4:
+                    return ANSI_HILITE + ANSI_WHITE
+                else:
+                    return ANSI_NORMAL + ANSI_WHITE
+            elif red > green and red > blue:
+                if background:
+                    return ANSI_BACK_RED
+                elif red >= 3:
+                    return ANSI_HILITE + ANSI_RED
+                else:
+                    return ANSI_NORMAL + ANSI_RED
+            elif red == green and red > blue:
+                if background:
+                    return ANSI_BACK_YELLOW
+                elif red >= 3:
+                    return ANSI_HILITE + ANSI_YELLOW
+                else:
+                    return ANSI_NORMAL + ANSI_YELLOW
+            elif red == blue and red > green:
+                if background:
+                    return ANSI_BACK_MAGENTA
+                elif red >= 3:
+                    return ANSI_HILITE + ANSI_MAGENTA
+                else:
+                    return ANSI_NORMAL + ANSI_MAGENTA
+            elif green > blue:
+                if background:
+                    return ANSI_BACK_GREEN
+                elif green >= 3:
+                    return ANSI_HILITE + ANSI_GREEN
+                else:
+                    return ANSI_NORMAL + ANSI_GREEN
+            elif green == blue:
+                if background:
+                    return ANSI_BACK_CYAN
+                elif green >= 3:
+                    return ANSI_HILITE + ANSI_CYAN
+                else:
+                    return ANSI_NORMAL + ANSI_CYAN
+            else:    # mostly blue
+                if background:
+                    return ANSI_BACK_BLUE
+                elif blue >= 3:
+                    return ANSI_HILITE + ANSI_BLUE
+                else:
+                    return ANSI_NORMAL + ANSI_BLUE
+
+    def strip_raw_codes(self, string):
+        """
+        Strips raw ANSI codes from a string.
+        """
+        return self.ansi_regex.sub("", string)
+
+    def strip_mxp(self, string):
+        """
+        Strips all MXP codes from a string.
+        """
+        return self.mxp_sub.sub(r'\2', string)
+
+    def parse_ansi(self, string, strip_ansi=False, xterm256=False, mxp=False):
+        """
+        Parses a string, subbing color codes according to
+        the stored mapping.
+
+        strip_ansi flag instead removes all ansi markup.
+
+        """
+        if hasattr(string, '_raw_string'):
+            if strip_ansi:
+                return string.clean()
+            else:
+                return string.raw()
+
+        if not string:
+            return ''
+
+        # check cached parsings
+        global _PARSE_CACHE
+        cachekey = "%s-%s-%s" % (string, strip_ansi, xterm256)
+        if cachekey in _PARSE_CACHE:
+            return _PARSE_CACHE[cachekey]
+
+        self.do_xterm256 = xterm256
+        self.do_mxp = mxp
+        in_string = utils.to_str(string)
+
+        # do string replacement
+        parsed_string =  ""
+        parts = self.ansi_escapes.split(in_string) + [" "]
+        for part, sep in zip(parts[::2], parts[1::2]):
+            pstring = self.xterm256_sub.sub(self.sub_xterm256, part)
+            pstring = self.ansi_sub.sub(self.sub_ansi, pstring)
+            parsed_string += "%s%s" % (pstring, sep[0].strip())
+
+        if strip_ansi:
+            # remove all ansi codes (including those manually
+            # inserted in string)
+            parsed_string = self.strip_mxp(parsed_string)
+            return self.strip_raw_codes(parsed_string)
+
+        if not mxp:
+            parsed_string = self.strip_mxp(parsed_string)
+
+         # cache and crop old cache
+        _PARSE_CACHE[cachekey] = parsed_string
+        if len(_PARSE_CACHE) > _PARSE_CACHE_SIZE:
+           _PARSE_CACHE.popitem(last=False)
+
+        return parsed_string
+    # MUX-style mappings %cr %cn etc
+
+    mux_ansi_map = [
+        (r'%cn', ANSI_NORMAL),
+        (r'%ch', ANSI_HILITE),
+        (r'%r', ANSI_RETURN),
+        (r'%R', ANSI_RETURN),
+        (r'%t', ANSI_TAB),
+        (r'%T', ANSI_TAB),
+        (r'%b', ANSI_SPACE),
+        (r'%B', ANSI_SPACE),
+        (r'%cf', ANSI_BLINK), # annoying and not supported by all clients
+        (r'%ci', ANSI_INVERSE),
+
+        (r'%cr', ANSI_RED),
+        (r'%cg', ANSI_GREEN),
+        (r'%cy', ANSI_YELLOW),
+        (r'%cb', ANSI_BLUE),
+        (r'%cm', ANSI_MAGENTA),
+        (r'%cc', ANSI_CYAN),
+        (r'%cw', ANSI_WHITE),
+        (r'%cx', ANSI_BLACK),
+
+        (r'%cR', ANSI_BACK_RED),
+        (r'%cG', ANSI_BACK_GREEN),
+        (r'%cY', ANSI_BACK_YELLOW),
+        (r'%cB', ANSI_BACK_BLUE),
+        (r'%cM', ANSI_BACK_MAGENTA),
+        (r'%cC', ANSI_BACK_CYAN),
+        (r'%cW', ANSI_BACK_WHITE),
+        (r'%cX', ANSI_BACK_BLACK)
+        ]
+
+    # Expanded mapping {r {n etc
+
+    hilite = ANSI_HILITE
+    normal = ANSI_NORMAL
+
+    ext_ansi_map = [
+        (r'{n', normal),                # reset
+        (r'{/', ANSI_RETURN),          # line break
+        (r'{-', ANSI_TAB),             # tab
+        (r'{_', ANSI_SPACE),           # space
+        (r'{*', ANSI_INVERSE),        # invert
+        (r'{^', ANSI_BLINK),          # blinking text (very annoying and not supported by all clients)
+
+        (r'{r', hilite + ANSI_RED),
+        (r'{g', hilite + ANSI_GREEN),
+        (r'{y', hilite + ANSI_YELLOW),
+        (r'{b', hilite + ANSI_BLUE),
+        (r'{m', hilite + ANSI_MAGENTA),
+        (r'{c', hilite + ANSI_CYAN),
+        (r'{w', hilite + ANSI_WHITE),  # pure white
+        (r'{x', hilite + ANSI_BLACK),  # dark grey
+
+        (r'{R', normal + ANSI_RED),
+        (r'{G', normal + ANSI_GREEN),
+        (r'{Y', normal + ANSI_YELLOW),
+        (r'{B', normal + ANSI_BLUE),
+        (r'{M', normal + ANSI_MAGENTA),
+        (r'{C', normal + ANSI_CYAN),
+        (r'{W', normal + ANSI_WHITE),  # light grey
+        (r'{X', normal + ANSI_BLACK),  # pure black
+
+        (r'{[r', ANSI_BACK_RED),
+        (r'{[g', ANSI_BACK_GREEN),
+        (r'{[y', ANSI_BACK_YELLOW),
+        (r'{[b', ANSI_BACK_BLUE),
+        (r'{[m', ANSI_BACK_MAGENTA),
+        (r'{[c', ANSI_BACK_CYAN),
+        (r'{[w', ANSI_BACK_WHITE),    # light grey background
+        (r'{[x', ANSI_BACK_BLACK)     # pure black background
+        ]
+
+    #ansi_map = mux_ansi_map + ext_ansi_map
+
+    # xterm256 {123, %c134. These are replaced directly by
+    # the sub_xterm256 method
+
+    xterm256_map = [
+        (r'%[0-5]{3}', ""),  # %123 - foreground colour
+        (r'%\[[0-5]{3}', ""),  # %[123 - background colour
+        (r'\{[0-5]{3}', ""),   # {123 - foreground colour
+        (r'\{\[[0-5]{3}', "")   # {[123 - background colour
+        ]
+
+    mxp_re = r'\{lc(.*?)\{lt(.*?)\{le'
+
+    # prepare regex matching
+    xterm256_sub = re.compile(r"|".join([tup[0] for tup in xterm256_map]), re.DOTALL)
+    ansi_sub = re.compile(r"|".join([re.escape(tup[0]) for tup in mux_ansi_map + ext_ansi_map]), re.DOTALL)
+    mxp_sub = re.compile(mxp_re, re.DOTALL)
+
+    # used by regex replacer to correctly map ansi sequences
+    ansi_map = dict(mux_ansi_map + ext_ansi_map)
+
+    # prepare matching ansi codes overall
+    ansi_regex = re.compile("\033\[[0-9;]+m")
+
+    # escapes - these double-chars will be replaced with a single
+    # instance of each
+    ansi_escapes = re.compile(r"(%s)" % "|".join(ANSI_ESCAPES), re.DOTALL)
+
+ANSI_PARSER = ANSIParser()
+
+
+#
+# Access function
+#
+
+def parse_ansi(string, strip_ansi=False, parser=ANSI_PARSER, xterm256=False, mxp=False):
+    """
+    Parses a string, subbing color codes as needed.
+
+    """
+    return parser.parse_ansi(string, strip_ansi=strip_ansi, xterm256=xterm256, mxp=mxp)
+
+
+def strip_raw_ansi(string, parser=ANSI_PARSER):
+    """
+    Remove raw ansi codes from string
+    """
+    return parser.strip_raw_codes(string)
+
+
+def raw(string):
+    """
+    Escapes a string into a form which won't be colorized by the ansi parser.
+    """
+    return string.replace('{', '{{').replace('%', '%%')
+
+
+def group(lst, n):
+    for i in range(0, len(lst), n):
+        val = lst[i:i+n]
+        if len(val) == n:
+            yield tuple(val)
+
+
+def _spacing_preflight(func):
+    """
+    This wrapper function is used to do some preflight checks on functions used
+    for padding ANSIStrings.
+    """
+    def wrapped(self, width, fillchar=None):
+        if fillchar is None:
+            fillchar = " "
+        if (len(fillchar) != 1) or (not isinstance(fillchar, basestring)):
+            raise TypeError("must be char, not %s" % type(fillchar))
+        if not isinstance(width, int):
+            raise TypeError("integer argument expected, got %s" % type(width))
+        difference = width - len(self)
+        if difference <= 0:
+            return self
+        return func(self, width, fillchar, difference)
+    return wrapped
+
+
+def _query_super(func_name):
+    """
+    Have the string class handle this with the cleaned string instead of
+    ANSIString.
+    """
+    def wrapped(self, *args, **kwargs):
+        return getattr(self.clean(), func_name)(*args, **kwargs)
+    return wrapped
+
+
+def _on_raw(func_name):
+    """
+    Like query_super, but makes the operation run on the raw string.
+    """
+    def wrapped(self, *args, **kwargs):
+        args = list(args)
+        try:
+            string = args.pop(0)
+            if hasattr(string, '_raw_string'):
+                args.insert(0, string.raw())
+            else:
+                args.insert(0, string)
+        except IndexError:
+            pass
+        result = getattr(self._raw_string, func_name)(*args, **kwargs)
+        if isinstance(result, basestring):
+            return ANSIString(result, decoded=True)
+        return result
+    return wrapped
+
+
+def _transform(func_name):
+    """
+    Some string functions, like those manipulating capital letters,
+    return a string the same length as the original. This function
+    allows us to do the same, replacing all the non-coded characters
+    with the resulting string.
+    """
+    def wrapped(self, *args, **kwargs):
+        replacement_string = _query_super(func_name)(self, *args, **kwargs)
+        to_string = []
+        char_counter = 0
+        for index in range(0, len(self._raw_string)):
+            if index in self._code_indexes:
+                to_string.append(self._raw_string[index])
+            elif index in self._char_indexes:
+                to_string.append(replacement_string[char_counter])
+                char_counter += 1
+        return ANSIString(
+            ''.join(to_string), decoded=True,
+            code_indexes=self._code_indexes, char_indexes=self._char_indexes,
+            clean_string=replacement_string)
+    return wrapped
+
+
+class ANSIMeta(type):
+    """
+    Many functions on ANSIString are just light wrappers around the unicode
+    base class. We apply them here, as part of the classes construction.
+    """
+    def __init__(cls, *args, **kwargs):
+        for func_name in [
+                'count', 'startswith', 'endswith', 'find', 'index', 'isalnum',
+                'isalpha', 'isdigit', 'islower', 'isspace', 'istitle', 'isupper',
+                'rfind', 'rindex', '__len__']:
+            setattr(cls, func_name, _query_super(func_name))
+        for func_name in [
+                '__mod__', 'expandtabs', 'decode', 'replace', 'format',
+                'encode']:
+            setattr(cls, func_name, _on_raw(func_name))
+        for func_name in [
+                'capitalize', 'translate', 'lower', 'upper', 'swapcase']:
+            setattr(cls, func_name, _transform(func_name))
+        super(ANSIMeta, cls).__init__(*args, **kwargs)
+
+
+class ANSIString(unicode):
+    """
+    String-like object that is aware of ANSI codes.
+
+    This isn't especially efficient, as it doesn't really have an
+    understanding of what the codes mean in order to eliminate
+    redundant characters. This could be made as an enhancement to ANSI_PARSER.
+
+    If one is going to use ANSIString, one should generally avoid converting
+    away from it until one is about to send information on the wire. This is
+    because escape sequences in the string may otherwise already be decoded,
+    and taken literally the second time around.
+
+    Please refer to the Metaclass, ANSIMeta, which is used to apply wrappers
+    for several of the methods that need not be defined directly here.
+    """
+    __metaclass__ = ANSIMeta
+
+    def __new__(cls, *args, **kwargs):
+        """
+        When creating a new ANSIString, you may use a custom parser that has
+        the same attributes as the standard one, and you may declare the
+        string to be handled as already decoded. It is important not to double
+        decode strings, as escapes can only be respected once.
+
+        Internally, ANSIString can also passes itself precached code/character
+        indexes and clean strings to avoid doing extra work when combining
+        ANSIStrings.
+        """
+        string = args[0]
+        if not isinstance(string, basestring):
+            string = to_str(string, force_string=True)
+        parser = kwargs.get('parser', ANSI_PARSER)
+        decoded = kwargs.get('decoded', False) or hasattr(string, '_raw_string')
+        code_indexes = kwargs.pop('code_indexes', None)
+        char_indexes = kwargs.pop('char_indexes', None)
+        clean_string = kwargs.pop('clean_string', None)
+        # All True, or All False, not just one.
+        checks = map(lambda x: x is None, [code_indexes, char_indexes, clean_string])
+        if not len(set(checks)) == 1:
+            raise ValueError("You must specify code_indexes, char_indexes, "
+                             "and clean_string together, or not at all.")
+        if not all(checks):
+            decoded = True
+        if not decoded:
+            # Completely new ANSI String
+            clean_string = to_unicode(parser.parse_ansi(string, strip_ansi=True))
+            string = parser.parse_ansi(string)
+        elif clean_string is not None:
+            # We have an explicit clean string.
+            pass
+        elif hasattr(string, '_clean_string'):
+            # It's already an ANSIString
+            clean_string = string._clean_string
+            code_indexes = string._code_indexes
+            char_indexes = string._char_indexes
+            string = string._raw_string
+        else:
+            # It's a string that has been pre-ansi decoded.
+            clean_string = parser.strip_raw_codes(string)
+
+        if not isinstance(string, unicode):
+            string = string.decode('utf-8')
+
+        ansi_string = super(ANSIString, cls).__new__(ANSIString, to_str(clean_string), "utf-8")
+        ansi_string._raw_string = string
+        ansi_string._clean_string = clean_string
+        ansi_string._code_indexes = code_indexes
+        ansi_string._char_indexes = char_indexes
+        return ansi_string
+
+    def __str__(self):
+        return self._raw_string.encode('utf-8')
+
+    def __unicode__(self):
+        """
+        Unfortunately, this is not called during print() statements due to a
+        bug in the Python interpreter. You can always do unicode() or str()
+        around the resulting ANSIString and print that.
+        """
+        return self._raw_string
+
+    def __repr__(self):
+        """
+        Let's make the repr the command that would actually be used to
+        construct this object, for convenience and reference.
+        """
+        return "ANSIString(%s, decoded=True)" % repr(self._raw_string)
+
+    def __init__(self, *_, **kwargs):
+        """
+        When the ANSIString is first initialized, a few internal variables
+        have to be set.
+
+        The first is the parser. It is possible to replace Evennia's standard
+        ANSI parser with one of your own syntax if you wish, so long as it
+        implements the same interface.
+
+        The second is the _raw_string. It should be noted that the ANSIStrings
+        are unicode based. This seemed more reasonable than basing it off of
+        the string class, because if someone were to use a unicode character,
+        the benefits of knowing the indexes of the ANSI characters would be
+        negated by the fact that a character within the string might require
+        more than one byte to be represented. The raw string is, then, a
+        unicode object rather than a true encoded string. If you need the
+        encoded string for sending over the wire, try using the .encode()
+        method.
+
+        The third thing to set is the _clean_string. This is a unicode object
+        that is devoid of all ANSI Escapes.
+
+        Finally, _code_indexes and _char_indexes are defined. These are lookup
+        tables for which characters in the raw string are related to ANSI
+        escapes, and which are for the readable text.
+        """
+        self.parser = kwargs.pop('parser', ANSI_PARSER)
+        super(ANSIString, self).__init__()
+        if self._code_indexes is None:
+            self._code_indexes, self._char_indexes = self._get_indexes()
+
+    @staticmethod
+    def _shifter(iterable, offset):
+        """
+        Takes a list of integers, and produces a new one incrementing all
+        by a number.
+        """
+        return [i + offset for i in iterable]
+
+    @classmethod
+    def _adder(cls, first, second):
+        """
+        Joins two ANSIStrings, preserving calculated info.
+        """
+
+        raw_string = first._raw_string + second._raw_string
+        clean_string = first._clean_string + second._clean_string
+        code_indexes = first._code_indexes[:]
+        char_indexes = first._char_indexes[:]
+        code_indexes.extend(
+            cls._shifter(second._code_indexes, len(first._raw_string)))
+        char_indexes.extend(
+            cls._shifter(second._code_indexes, len(first._raw_string)))
+        return ANSIString(raw_string, code_indexes=code_indexes,
+                          char_indexes=char_indexes,
+                          clean_string=clean_string)
+
+    def __add__(self, other):
+        """
+        We have to be careful when adding two strings not to reprocess things
+        that don't need to be reprocessed, lest we end up with escapes being
+        interpreted literally.
+        """
+        if not isinstance(other, basestring):
+            return NotImplemented
+        if not isinstance(other, ANSIString):
+            other = ANSIString(other)
+        return self._adder(self, other)
+
+    def __radd__(self, other):
+        """
+        Likewise, if we're on the other end.
+        """
+        if not isinstance(other, basestring):
+            return NotImplemented
+        if not isinstance(other, ANSIString):
+            other = ANSIString(other)
+        return self._adder(other, self)
+
+    def __getslice__(self, i, j):
+        """
+        This function is deprecated, so we just make it call the proper
+        function.
+        """
+        return self.__getitem__(slice(i, j))
+
+    def _slice(self, slc):
+        """
+        This function takes a slice() object.
+
+        Slices have to be handled specially. Not only are they able to specify
+        a start and end with [x:y], but many forget that they can also specify
+        an interval with [x:y:z]. As a result, not only do we have to track
+        the ANSI Escapes that have played before the start of the slice, we
+        must also replay any in these intervals, should the exist.
+
+        Thankfully, slicing the _char_indexes table gives us the actual
+        indexes that need slicing in the raw string. We can check between
+        those indexes to figure out what escape characters need to be
+        replayed.
+        """
+        slice_indexes = self._char_indexes[slc]
+        # If it's the end of the string, we need to append final color codes.
+        if not slice_indexes:
+            return ANSIString('')
+        try:
+            string = self[slc.start]._raw_string
+        except IndexError:
+            return ANSIString('')
+        last_mark = slice_indexes[0]
+        # Check between the slice intervals for escape sequences.
+        i = None
+        for i in slice_indexes[1:]:
+            for index in xrange(last_mark, i):
+                if index in self._code_indexes:
+                    string += self._raw_string[index]
+            last_mark = i
+            try:
+                string += self._raw_string[i]
+            except IndexError:
+                pass
+        if i is not None:
+            append_tail = self._get_interleving(self._char_indexes.index(i) + 1)
+        else:
+            append_tail = ''
+        return ANSIString(string + append_tail, decoded=True)
+
+    def __getitem__(self, item):
+        """
+        Gateway for slices and getting specific indexes in the ANSIString. If
+        this is a regexable ANSIString, it will get the data from the raw
+        string instead, bypassing ANSIString's intelligent escape skipping,
+        for reasons explained in the __new__ method's docstring.
+        """
+        if isinstance(item, slice):
+            # Slices must be handled specially.
+            return self._slice(item)
+        try:
+            self._char_indexes[item]
+        except IndexError:
+            raise IndexError("ANSIString Index out of range")
+        # Get character codes after the index as well.
+        if self._char_indexes[-1] == self._char_indexes[item]:
+            append_tail = self._get_interleving(item + 1)
+        else:
+            append_tail = ''
+        item = self._char_indexes[item]
+
+        clean = self._raw_string[item]
+        result = ''
+        # Get the character they're after, and replay all escape sequences
+        # previous to it.
+        for index in xrange(0, item + 1):
+            if index in self._code_indexes:
+                result += self._raw_string[index]
+        return ANSIString(result + clean + append_tail, decoded=True)
+
+    def clean(self):
+        """
+        Return a unicode object without the ANSI escapes.
+        """
+        return self._clean_string
+
+    def raw(self):
+        """
+        Return a unicode object with the ANSI escapes.
+        """
+        return self._raw_string
+
+    def partition(self, sep, reverse=False):
+        """
+        Similar to split, but always creates a tuple with three items:
+        1. The part before the separator
+        2. The separator itself.
+        3. The part after.
+
+        We use the same techniques we used in split() to make sure each are
+        colored.
+        """
+        if hasattr(sep, '_clean_string'):
+            sep = sep.clean()
+        if reverse:
+            parent_result = self._clean_string.rpartition(sep)
+        else:
+            parent_result = self._clean_string.partition(sep)
+        current_index = 0
+        result = tuple()
+        for section in parent_result:
+            result += (self[current_index:current_index + len(section)],)
+            current_index += len(section)
+        return result
+
+    def _get_indexes(self):
+        """
+        Two tables need to be made, one which contains the indexes of all
+        readable characters, and one which contains the indexes of all ANSI
+        escapes. It's important to remember that ANSI escapes require more
+        that one character at a time, though no readable character needs more
+        than one character, since the unicode base class abstracts that away
+        from us. However, several readable characters can be placed in a row.
+
+        We must use regexes here to figure out where all the escape sequences
+        are hiding in the string. Then we use the ranges of their starts and
+        ends to create a final, comprehensive list of all indexes which are
+        dedicated to code, and all dedicated to text.
+
+        It's possible that only one of these tables is actually needed, the
+        other assumed to be what isn't in the first.
+        """
+
+        code_indexes = []
+        for match in self.parser.ansi_regex.finditer(self._raw_string):
+            code_indexes.extend(range(match.start(), match.end()))
+        if not code_indexes:
+            # Plain string, no ANSI codes.
+            return code_indexes, range(0, len(self._raw_string))
+        # all indexes not occupied by ansi codes are normal characters
+        char_indexes = [i for i in range(len(self._raw_string)) if i not in code_indexes]
+        return code_indexes, char_indexes
+
+    def _get_interleving(self, index):
+        """
+        Get the code characters from the given slice end to the next
+        character.
+        """
+        try:
+            index = self._char_indexes[index - 1]
+        except IndexError:
+            return ''
+        s = ''
+        while True:
+            index += 1
+            if index in self._char_indexes:
+                break
+            elif index in self._code_indexes:
+                s += self._raw_string[index]
+            else:
+                break
+        return s
+
+    def split(self, by, maxsplit=-1):
+        """
+        Stolen from PyPy's pure Python string implementation, tweaked for
+        ANSIString.
+
+        PyPy is distributed under the MIT licence.
+        http://opensource.org/licenses/MIT
+        """
+        bylen = len(by)
+        if bylen == 0:
+            raise ValueError("empty separator")
+
+        res = []
+        start = 0
+        while maxsplit != 0:
+            next = self._clean_string.find(by, start)
+            if next < 0:
+                break
+            # Get character codes after the index as well.
+            res.append(self[start:next])
+            start = next + bylen
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        res.append(self[start:len(self)])
+        return res
+
+    def __mul__(self, other):
+        """
+        Multiplication method. Implemented for performance reasons.
+        """
+        if not isinstance(other, int):
+            return NotImplemented
+        raw_string = self._raw_string * other
+        clean_string = self._clean_string * other
+        code_indexes = self._code_indexes[:]
+        char_indexes = self._char_indexes[:]
+        for i in range(1, other + 1):
+            code_indexes.extend(
+                self._shifter(self._code_indexes, i * len(self._raw_string)))
+            char_indexes.extend(
+                self._shifter(self._char_indexes, i * len(self._raw_string)))
+        return ANSIString(
+            raw_string, code_indexes=code_indexes, char_indexes=char_indexes,
+            clean_string=clean_string)
+
+    def __rmul__(self, other):
+        return self.__mul__(other)
+
+    def rsplit(self, by, maxsplit=-1):
+        """
+        Stolen from PyPy's pure Python string implementation, tweaked for
+        ANSIString.
+
+        PyPy is distributed under the MIT licence.
+        http://opensource.org/licenses/MIT
+        """
+        res = []
+        end = len(self)
+        bylen = len(by)
+        if bylen == 0:
+            raise ValueError("empty separator")
+
+        while maxsplit != 0:
+            next = self._clean_string.rfind(by, 0, end)
+            if next < 0:
+                break
+            # Get character codes after the index as well.
+            res.append(self[next+bylen:end])
+            end = next
+            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
+
+        res.append(self[:end])
+        res.reverse()
+        return res
+
+    def join(self, iterable):
+        """
+        Joins together strings in an iterable.
+        """
+        result = ANSIString('')
+        last_item = None
+        for item in iterable:
+            if last_item is not None:
+                result += self._raw_string
+            if not isinstance(item, ANSIString):
+                item = ANSIString(item)
+            result += item
+            last_item = item
+        return result
+
+    def _filler(self, char, amount):
+        """
+        Generate a line of characters in a more efficient way than just adding
+        ANSIStrings.
+        """
+        if not isinstance(char, ANSIString):
+            line = char * amount
+            return ANSIString(
+                char * amount, code_indexes=[], char_indexes=range(0, len(line)),
+                clean_string=char)
+        try:
+            start = char._code_indexes[0]
+        except IndexError:
+            start = None
+        end = char._char_indexes[0]
+        prefix = char._raw_string[start:end]
+        postfix = char._raw_string[end + 1:]
+        line = char._clean_string * amount
+        code_indexes = [i for i in range(0, len(prefix))]
+        length = len(prefix) + len(line)
+        code_indexes.extend([i for i in range(length, length + len(postfix))])
+        char_indexes = self._shifter(xrange(0, len(line)), len(prefix))
+        raw_string = prefix + line + postfix
+        return ANSIString(
+            raw_string, clean_string=line, char_indexes=char_indexes,
+            code_indexes=code_indexes)
+
+    @_spacing_preflight
+    def center(self, width, fillchar, difference):
+        """
+        Center some text with some spaces padding both sides.
+        """
+        remainder = difference % 2
+        difference /= 2
+        spacing = self._filler(fillchar, difference)
+        result = spacing + self + spacing + self._filler(fillchar, remainder)
+        return result
+
+    @_spacing_preflight
+    def ljust(self, width, fillchar, difference):
+        """
+        Left justify some text.
+        """
+        return self + self._filler(fillchar, difference)
+
+    @_spacing_preflight
+    def rjust(self, width, fillchar, difference):
+        """
+        Right justify some text.
+        """
+        return self._filler(fillchar, difference) + self

lib/utils/batchprocessors.py

@@ -0,0 +1,418 @@
+"""
+This file contains the core methods for the Batch-command- and
+Batch-code-processors respectively. In short, these are two different
+ways to build a game world using a normal text-editor without having
+to do so 'on the fly' in-game. They also serve as an automatic backup
+so you can quickly recreate a world also after a server reset. The
+functions in this module is meant to form the backbone of a system
+called and accessed through game commands.
+
+The Batch-command processor is the simplest. It simply runs a list of
+in-game commands in sequence by reading them from a text file. The
+advantage of this is that the builder only need to remember the normal
+in-game commands. They are also executing with full permission checks
+etc, making it relatively safe for builders to use. The drawback is
+that in-game there is really a builder-character walking around
+building things, and it can be important to create rooms and objects
+in the right order, so the character can move between them. Also
+objects that affects players (such as mobs, dark rooms etc) will
+affect the building character too, requiring extra care to turn
+off/on.
+
+The Batch-code processor is a more advanced system that accepts full
+Python code, executing in chunks. The advantage of this is much more
+power; practically anything imaginable can be coded and handled using
+the batch-code processor. There is no in-game character that moves and
+that can be affected by what is being built - the database is
+populated on the fly. The drawback is safety and entry threshold - the
+code is executed as would any server code, without mud-specific
+permission checks and you have full access to modifying objects
+etc. You also need to know Python and Evennia's API. Hence it's
+recommended that the batch-code processor is limited only to
+superusers or highly trusted staff.
+
+
+=======================================================================
+
+Batch-command processor file syntax
+
+The batch-command processor accepts 'batchcommand files' e.g
+'batch.ev', containing a sequence of valid evennia commands in a
+simple format. The engine runs each command in sequence, as if they
+had been run at the game prompt.
+
+Each evennia command must be delimited by a line comment to mark its
+end.
+
+#INSERT path.batchcmdfile - this as the first entry on a line will
+      import and run a batch.ev file in this position, as if it was
+      written in this file.
+
+This way entire game worlds can be created and planned offline; it is
+especially useful in order to create long room descriptions where a
+real offline text editor is often much better than any online text
+editor or prompt.
+
+Example of batch.ev file:
+----------------------------
+
+# batch file
+# all lines starting with # are comments; they also indicate
+# that a command definition is over.
+
+@create box
+
+# this comment ends the @create command.
+
+@set box/desc = A large box.
+
+Inside are some scattered piles of clothing.
+
+
+It seems the bottom of the box is a bit loose.
+
+# Again, this comment indicates the @set command is over. Note how
+# the description could be freely added. Excess whitespace on a line
+# is ignored.  An empty line in the command definition is parsed as a \n
+# (so two empty lines becomes a new paragraph).
+
+@teleport #221
+
+# (Assuming #221 is a warehouse or something.)
+# (remember, this comment ends the @teleport command! Don'f forget it)
+
+# Example of importing another file at this point.
+#IMPORT examples.batch
+
+@drop box
+
+# Done, the box is in the warehouse! (this last comment is not necessary to
+# close the @drop command since it's the end of the file)
+-------------------------
+
+An example batch file is game/gamesrc/commands/examples/batch_example.ev.
+
+
+==========================================================================
+
+
+Batch-code processor file syntax
+
+The Batch-code processor accepts full python modules (e.g. "batch.py")
+that looks identical to normal Python files with a few exceptions that
+allows them to the executed in blocks. This way of working assures a
+sequential execution of the file and allows for features like stepping
+from block to block (without executing those coming before), as well
+as automatic deletion of created objects etc. You can however also run
+a batch-code python file directly using Python (and can also be de).
+
+Code blocks are separated by python comments starting with special
+code words.
+
+#HEADER - this denotes commands global to the entire file, such as
+          import statements and global variables. They will
+          automatically be pasted at the top of all code
+          blocks. Observe that changes to these variables made in one
+          block is not preserved between blocks!
+#CODE
+#CODE (info)
+#CODE (info) objname1, objname1, ... -
+           This designates a code block that will be executed like a
+           stand-alone piece of code together with any #HEADER
+           defined. (info) text is used by the interactive mode to
+           display info about the node to run.  <objname>s mark the
+           (variable-)names of objects created in the code, and which
+           may be auto-deleted by the processor if desired (such as
+           when debugging the script). E.g., if the code contains the
+           command myobj = create.create_object(...), you could put
+           'myobj' in the #CODE header regardless of what the created
+           object is actually called in-game.
+#INSERT path.filename - This imports another batch_code.py file and
+          runs it in the given position.  paths are given as python
+          path. The inserted file will retain its own HEADERs which
+          will not be mixed with the HEADERs of the file importing
+          this file.
+
+The following variables are automatically made available for the script:
+
+caller - the object executing the script
+
+Example batch.py file
+-----------------------------------
+
+#HEADER
+
+import traceback
+from django.config import settings
+from src.utils import create
+from game.gamesrc.typeclasses import basetypes
+
+GOLD = 10
+
+#CODE obj, obj2
+
+obj = create.create_object(basetypes.Object)
+obj2 = create.create_object(basetypes.Object)
+obj.location = caller.location
+obj.db.gold = GOLD
+caller.msg("The object was created!")
+
+#INSERT another_batch_file
+
+#CODE
+
+script = create.create_script()
+
+"""
+
+import re
+import codecs
+import traceback
+import sys
+#from traceback import format_exc
+from django.conf import settings
+from src.utils import utils
+#from game import settings as settings_module
+
+ENCODINGS = settings.ENCODINGS
+CODE_INFO_HEADER = re.compile(r"\(.*?\)")
+
+RE_INSERT = re.compile(r"^\#INSERT (.*)", re.MULTILINE)
+RE_CLEANBLOCK = re.compile(r"^\#.*?$|^\s*$", re.MULTILINE)
+RE_CMD_SPLIT = re.compile(r"^\#.*?$", re.MULTILINE)
+RE_CODE_SPLIT = re.compile(r"(^\#CODE.*?$|^\#HEADER)$", re.MULTILINE)
+
+
+#------------------------------------------------------------
+# Helper function
+#------------------------------------------------------------
+
+def read_batchfile(pythonpath, file_ending='.py'):
+    """
+    This reads the contents of a batch-file.
+    Filename is considered to be a python path to a batch file
+    relative the directory specified in settings.py.
+
+    file_ending specify which batchfile ending should be
+    assumed (.ev or .py). The ending should not be included
+    in the python path.
+    """
+
+    # open the file
+    if pythonpath and not (pythonpath.startswith('src.') or pythonpath.startswith('game.')
+                           or pythonpath.startswith('contrib.')):
+        abspaths = []
+        for basepath in settings.BASE_BATCHPROCESS_PATHS:
+            abspaths.append(utils.pypath_to_realpath("%s.%s" % (basepath, pythonpath), file_ending))
+    else:
+        abspaths = [utils.pypath_to_realpath(pythonpath, file_ending)]
+    text, fobj  = None, None
+    fileerr, decoderr = [], []
+    for abspath in abspaths:
+        # try different paths, until we get a match
+        # we read the file directly into unicode.
+        for file_encoding in ENCODINGS:
+            # try different encodings, in order
+            try:
+                fobj = codecs.open(abspath, 'r', encoding=file_encoding)
+                text = fobj.read()
+            except IOError, e:
+                # could not find the file
+                fileerr.append(str(e))
+                break
+            except (ValueError, UnicodeDecodeError), e:
+                # this means an encoding error; try another encoding
+                decoderr.append(str(e))
+                continue
+            break
+    if not fobj:
+        raise IOError("\n".join(fileerr))
+    if not text:
+        raise UnicodeDecodeError("\n".join(decoderr))
+
+    return text
+
+
+#------------------------------------------------------------
+#
+# Batch-command processor
+#
+#------------------------------------------------------------
+
+class BatchCommandProcessor(object):
+    """
+    This class implements a batch-command processor.
+
+    """
+    def parse_file(self, pythonpath):
+        """
+        This parses the lines of a batchfile according to the following
+        rules:
+          1) # at the beginning of a line marks the end of the command before
+               it. It is also a comment and any number of # can exist on
+               subsequent lines (but not inside comments).
+          2) #INSERT at the beginning of a line imports another
+             batch-cmd file file and pastes it into the batch file as if
+             it was written there.
+          3) Commands are placed alone at the beginning of a line and their
+             arguments are considered to be everything following (on any
+             number of lines) until the next comment line beginning with #.
+          4) Newlines are ignored in command definitions
+          5) A completely empty line in a command line definition is condered
+             a newline (so two empty lines is a paragraph).
+          6) Excess spaces and indents inside arguments are stripped.
+
+        """
+
+        text = "".join(read_batchfile(pythonpath, file_ending='.ev'))
+
+        def replace_insert(match):
+            "Map replace entries"
+            return "\#\n".join(self.parse_file(match.group(1)))
+
+        # insert commands from inserted files
+        text = RE_INSERT.sub(replace_insert, text)
+        #text = re.sub(r"^\#INSERT (.*?)", replace_insert, text, flags=re.MULTILINE)
+        # get all commands
+        commands = RE_CMD_SPLIT.split(text)
+        #commands = re.split(r"^\#.*?$", text, flags=re.MULTILINE)
+        #remove eventual newline at the end of commands
+        commands = [c.strip('\r\n') for c in commands]
+        commands = [c for c in commands if c]
+
+        return commands
+
+
+#------------------------------------------------------------
+#
+# Batch-code processor
+#
+#------------------------------------------------------------
+
+def tb_filename(tb):
+    "Helper to get filename from traceback"
+    return tb.tb_frame.f_code.co_filename
+
+
+def tb_iter(tb):
+    while tb is not None:
+        yield tb
+        tb = tb.tb_next
+
+
+class BatchCodeProcessor(object):
+    """
+    This implements a batch-code processor
+
+    """
+
+    def parse_file(self, pythonpath, debug=False):
+        """
+        This parses the lines of a batchfile according to the following
+        rules:
+
+        1) Lines starting with #HEADER starts a header block (ends other blocks)
+        2) Lines starting with #CODE begins a code block (ends other blocks)
+        3) #CODE headers may be of the following form:
+                              #CODE (info) objname, objname2, ...
+        4) Lines starting with #INSERT are on form #INSERT filename.
+        3) All lines outside blocks are stripped.
+        4) All excess whitespace beginning/ending a block is stripped.
+
+        """
+
+        text = "".join(read_batchfile(pythonpath, file_ending='.py'))
+
+        def clean_block(text):
+            text = RE_CLEANBLOCK.sub("", text)
+            #text = re.sub(r"^\#.*?$|^\s*$", "", text, flags=re.MULTILINE)
+            return "\n".join([line for line in text.split("\n") if line])
+
+        def replace_insert(match):
+            "Map replace entries"
+            return "\#\n".join(self.parse_file(match.group(1)))
+
+        text = RE_INSERT.sub(replace_insert, text)
+        #text = re.sub(r"^\#INSERT (.*?)", replace_insert, text, flags=re.MULTILINE)
+        blocks = RE_CODE_SPLIT.split(text)
+        #blocks = re.split(r"(^\#CODE.*?$|^\#HEADER)$", text, flags=re.MULTILINE)
+        headers = []
+        codes = [] # list of tuples (code, info, objtuple)
+        if blocks:
+            if blocks[0]:
+                # the first block is either empty or an unmarked code block
+                code = clean_block(blocks.pop(0))
+                if code:
+                    codes.append((code, ""))
+            iblock = 0
+            for block in blocks[::2]:
+                # loop over every second component; these are the #CODE/#HEADERs
+                if block.startswith("#HEADER"):
+                    headers.append(clean_block(blocks[iblock + 1]))
+                elif block.startswith("#CODE"):
+                    match = re.search(r"\(.*?\)", block)
+                    info = match.group() if match else ""
+                    objs = []
+                    if debug:
+                        # insert auto-delete lines into code
+                        objs = block[match.end():].split(",")
+                        objs = ["# added by Evennia's debug mode\n%s.delete()" % obj.strip() for obj in objs if obj]
+                    # build the code block
+                    code = "\n".join([clean_block(blocks[iblock + 1])] + objs)
+                    if code:
+                        codes.append((code, info))
+                iblock += 2
+
+        # join the headers together to one header
+        headers = "\n".join(headers)
+        if codes:
+            # add the headers at the top of each non-empty block
+            codes = ["%s\n%s\n%s" % ("#CODE %s: " % tup[1], headers, tup[0]) for tup in codes if tup[0]]
+        else:
+            codes = ["#CODE: \n" + headers]
+        return codes
+
+
+    def code_exec(self, code, extra_environ=None, debug=False):
+        """
+        Execute a single code block, including imports and appending global vars
+
+        extra_environ - dict with environment variables
+        """
+        # define the execution environment
+        environdict = {"settings_module": settings}
+        environ = "settings_module.configure()"
+        if extra_environ:
+            for key, value in extra_environ.items():
+                environdict[key] = value
+
+        # initializing the django settings at the top of code
+        code = "# auto-added by Evennia\n" \
+               "try: %s\n" \
+               "except RuntimeError: pass\n" \
+               "finally: del settings_module\n%s" % (environ, code)
+
+        # execute the block
+        try:
+            exec(code, environdict)
+        except Exception:
+            etype, value, tb = sys.exc_info()
+
+            fname = tb_filename(tb)
+            for tb in tb_iter(tb):
+                if fname != tb_filename(tb):
+                    break
+            lineno = tb.tb_lineno - 1
+            err = ""
+            for iline, line in enumerate(code.split("\n")):
+                if iline == lineno:
+                    err += "\n{w%02i{n: %s" % (iline + 1, line)
+                elif lineno - 5 < iline < lineno + 5:
+                    err += "\n%02i: %s" % (iline + 1, line)
+
+            err += "\n".join(traceback.format_exception(etype, value, tb))
+            return err
+        return None
+
+BATCHCMD = BatchCommandProcessor()
+BATCHCODE = BatchCodeProcessor()

lib/utils/create.py

@@ -0,0 +1,382 @@
+"""
+This module gathers all the essential database-creation
+functions for the game engine's various object types.
+
+Only objects created 'stand-alone' are in here, e.g. object Attributes
+are always created directly through their respective objects.
+
+Each creation_* function also has an alias named for the entity being
+created, such as create_object() and object().  This is for
+consistency with the utils.search module and allows you to do the
+shorter "create.object()".
+
+The respective object managers hold more methods for manipulating and
+searching objects already existing in the database.
+
+Models covered:
+ Objects
+ Scripts
+ Help
+ Message
+ Channel
+ Players
+"""
+from django.conf import settings
+from django.db import IntegrityError
+from django.utils import timezone
+from src.utils import logger
+from src.utils.utils import make_iter, class_from_module, dbid_to_obj
+
+# delayed imports
+_User = None
+_ObjectDB = None
+_Object = None
+_Script = None
+_ScriptDB = None
+_HelpEntry = None
+_Msg = None
+_Player = None
+_PlayerDB = None
+_to_object = None
+_ChannelDB = None
+_channelhandler = None
+
+
+# limit symbol import from API
+__all__ = ("create_object", "create_script", "create_help_entry",
+           "create_message", "create_channel", "create_player")
+
+_GA = object.__getattribute__
+
+#
+# Game Object creation
+#
+
+def create_object(typeclass=None, key=None, location=None,
+                  home=None, permissions=None, locks=None,
+                  aliases=None, destination=None, report_to=None, nohome=False):
+    """
+
+    Create a new in-game object.
+
+    keywords:
+        typeclass - class or python path to a typeclass
+        key - name of the new object. If not set, a name of #dbref will be set.
+        home - obj or #dbref to use as the object's home location
+        permissions - a comma-separated string of permissions
+        locks - one or more lockstrings, separated by semicolons
+        aliases - a list of alternative keys
+        destination - obj or #dbref to use as an Exit's target
+
+        nohome - this allows the creation of objects without a default home location;
+                 only used when creating the default location itself or during unittests
+    """
+    global _ObjectDB
+    if not _ObjectDB:
+        from src.objects.models import ObjectDB as _ObjectDB
+
+
+    typeclass = typeclass if typeclass else settings.BASE_OBJECT_TYPECLASS
+
+    if isinstance(typeclass, basestring):
+        # a path is given. Load the actual typeclass
+        typeclass = class_from_module(typeclass, settings.OBJECT_TYPECLASS_PATHS)
+
+    # Setup input for the create command. We use ObjectDB as baseclass here
+    # to give us maximum freedom (the typeclasses will load
+    # correctly when each object is recovered).
+
+    location = dbid_to_obj(location, _ObjectDB)
+    destination = dbid_to_obj(destination, _ObjectDB)
+    home = dbid_to_obj(home, _ObjectDB)
+    if not home:
+        try:
+            home = dbid_to_obj(settings.DEFAULT_HOME, _ObjectDB) if not nohome else None
+        except _ObjectDB.DoesNotExist:
+            raise _ObjectDB.DoesNotExist("settings.DEFAULT_HOME (= '%s') does not exist, or the setting is malformed." %
+                                         settings.DEFAULT_HOME)
+
+    # create new instance
+    new_object = typeclass(db_key=key, db_location=location,
+                              db_destination=destination, db_home=home,
+                              db_typeclass_path=typeclass.path)
+    # store the call signature for the signal
+    new_object._createdict = {"key":key, "location":location, "destination":destination,
+                              "home":home, "typeclass":typeclass.path, "permissions":permissions,
+                              "locks":locks, "aliases":aliases, "destination":destination,
+                              "report_to":report_to, "nohome":nohome}
+    # this will trigger the save signal which in turn calls the
+    # at_first_save hook on the typeclass, where the _createdict can be
+    # used.
+    new_object.save()
+    return new_object
+
+#alias for create_object
+object = create_object
+
+
+#
+# Script creation
+#
+
+def create_script(typeclass, key=None, obj=None, player=None, locks=None,
+                  interval=None, start_delay=None, repeats=None,
+                  persistent=None, autostart=True, report_to=None):
+    """
+    Create a new script. All scripts are a combination
+    of a database object that communicates with the
+    database, and an typeclass that 'decorates' the
+    database object into being different types of scripts.
+    It's behaviour is similar to the game objects except
+    scripts has a time component and are more limited in
+    scope.
+
+    Argument 'typeclass' can be either an actual
+    typeclass object or a python path to such an object.
+    Only set key here if you want a unique name for this
+    particular script (set it in config to give
+    same key to all scripts of the same type). Set obj
+    to tie this script to a particular object.
+
+    See src.scripts.manager for methods to manipulate existing
+    scripts in the database.
+
+    report_to is an obtional object to receive error messages.
+              If report_to is not set, an Exception with the
+              error will be raised. If set, this method will
+              return None upon errors.
+    """
+    global _ScriptDB
+    if not _ScriptDB:
+        from src.scripts.models import ScriptDB as _ScriptDB
+
+    typeclass = typeclass if typeclass else settings.BASE_SCRIPT_TYPECLASS
+
+    if isinstance(typeclass, basestring):
+        # a path is given. Load the actual typeclass
+        typeclass = class_from_module(typeclass, settings.SCRIPT_TYPECLASS_PATHS)
+
+    # validate input
+    kwarg = {}
+    if key: kwarg["db_key"] = key
+    if player: kwarg["db_player"] = dbid_to_obj(player, _ScriptDB)
+    if obj: kwarg["db_obj"] = dbid_to_obj(obj, _ScriptDB)
+    if interval: kwarg["db_interval"] = interval
+    if start_delay: kwarg["db_start_delay"] = start_delay
+    if repeats: kwarg["db_repeats"] = repeats
+    if persistent: kwarg["db_persistent"] = persistent
+
+    # create new instance
+    new_script = typeclass(**kwarg)
+
+    # store the call signature for the signal
+    new_script._createdict = {"key":key, "obj":obj, "player":player,
+                              "locks":locks, "interval":interval,
+                              "start_delay":start_delay, "repeats":repeats,
+                              "persistent":persistent, "autostart":autostart,
+                              "report_to":report_to}
+
+    # this will trigger the save signal which in turn calls the
+    # at_first_save hook on the tyepclass, where the _createdict
+    # can be used.
+    new_script.save()
+    return new_script
+
+#alias
+script = create_script
+
+
+#
+# Help entry creation
+#
+
+def create_help_entry(key, entrytext, category="General", locks=None):
+    """
+    Create a static help entry in the help database. Note that Command
+    help entries are dynamic and directly taken from the __doc__ entries
+    of the command. The database-stored help entries are intended for more
+    general help on the game, more extensive info, in-game setting information
+    and so on.
+    """
+    global _HelpEntry
+    if not _HelpEntry:
+        from src.help.models import HelpEntry as _HelpEntry
+
+    try:
+        new_help = _HelpEntry()
+        new_help.key = key
+        new_help.entrytext = entrytext
+        new_help.help_category = category
+        if locks:
+            new_help.locks.add(locks)
+        new_help.save()
+        return new_help
+    except IntegrityError:
+        string = "Could not add help entry: key '%s' already exists." % key
+        logger.log_errmsg(string)
+        return None
+    except Exception:
+        logger.log_trace()
+        return None
+# alias
+help_entry = create_help_entry
+
+
+#
+# Comm system methods
+#
+
+def create_message(senderobj, message, channels=None,
+                   receivers=None, locks=None, header=None):
+    """
+    Create a new communication message. Msgs are used for all
+    player-to-player communication, both between individual players
+    and over channels.
+    senderobj - the player sending the message. This must be the actual object.
+    message - text with the message. Eventual headers, titles etc
+              should all be included in this text string. Formatting
+              will be retained.
+    channels - a channel or a list of channels to send to. The channels
+             may be actual channel objects or their unique key strings.
+    receivers - a player to send to, or a list of them. May be Player objects
+               or playernames.
+    locks - lock definition string
+    header - mime-type or other optional information for the message
+
+    The Comm system is created very open-ended, so it's fully possible
+    to let a message both go to several channels and to several receivers
+    at the same time, it's up to the command definitions to limit this as
+    desired.
+    """
+    global _Msg
+    if not _Msg:
+        from src.comms.models import Msg as _Msg
+    if not message:
+        # we don't allow empty messages.
+        return
+    new_message = _Msg(db_message=message)
+    new_message.save()
+    for sender in make_iter(senderobj):
+        new_message.senders = sender
+    new_message.header = header
+    for channel in make_iter(channels):
+        new_message.channels = channel
+    for receiver in make_iter(receivers):
+        new_message.receivers = receiver
+    if locks:
+        new_message.locks.add(locks)
+    new_message.save()
+    return new_message
+message = create_message
+
+
+def create_channel(key, aliases=None, desc=None,
+                   locks=None, keep_log=True,
+                   typeclass=None):
+    """
+    Create A communication Channel. A Channel serves as a central
+    hub for distributing Msgs to groups of people without
+    specifying the receivers explicitly. Instead players may
+    'connect' to the channel and follow the flow of messages. By
+    default the channel allows access to all old messages, but
+    this can be turned off with the keep_log switch.
+
+    key - this must be unique.
+    aliases - list of alternative (likely shorter) keynames.
+    locks - lock string definitions
+    """
+    typeclass = typeclass if typeclass else settings.BASE_CHANNEL_TYPECLASS
+
+    if isinstance(typeclass, basestring):
+        # a path is given. Load the actual typeclass
+        typeclass = class_from_module(typeclass, settings.CHANNEL_TYPECLASS_PATHS)
+
+    # create new instance
+    new_channel = typeclass(db_key=key)
+
+    # store call signature for the signal
+    new_channel._createdict = {"key":key, "aliases":aliases,
+            "desc":desc, "locks":locks, "keep_log":keep_log}
+
+    # this will trigger the save signal which in turn calls the
+    # at_first_save hook on the typeclass, where the _createdict can be
+    # used.
+    new_channel.save()
+    return new_channel
+
+channel = create_channel
+
+
+
+#
+# Player creation methods
+#
+
+def create_player(key, email, password,
+                  typeclass=None,
+                  is_superuser=False,
+                  locks=None, permissions=None,
+                  report_to=None):
+
+    """
+    This creates a new player.
+
+    key - the player's name. This should be unique.
+    email - email on valid addr@addr.domain form.
+    password - password in cleartext
+    is_superuser - wether or not this player is to be a superuser
+    locks - lockstring
+    permission - list of permissions
+    report_to - an object with a msg() method to report errors to. If
+                not given, errors will be logged.
+
+    Will return the Player-typeclass or None/raise Exception if the
+    Typeclass given failed to load.
+
+    Concerning is_superuser:
+     Usually only the server admin should need to be superuser, all
+     other access levels can be handled with more fine-grained
+     permissions or groups. A superuser bypasses all lock checking
+     operations and is thus not suitable for play-testing the game.
+
+    """
+    global _PlayerDB
+    if not _PlayerDB:
+        from src.players.models import PlayerDB as _PlayerDB
+
+    typeclass = typeclass if typeclass else settings.BASE_PLAYER_TYPECLASS
+
+    if isinstance(typeclass, basestring):
+        # a path is given. Load the actual typeclass.
+        typeclass = class_from_module(typeclass, settings.OBJECT_TYPECLASS_PATHS)
+
+    # setup input for the create command. We use PlayerDB as baseclass
+    # here to give us maximum freedom (the typeclasses will load
+    # correctly when each object is recovered).
+
+    if not email:
+        email = "dummy@dummy.com"
+    if _PlayerDB.objects.filter(username__iexact=key):
+        raise ValueError("A Player with the name '%s' already exists." % key)
+
+    # this handles a given dbref-relocate to a player.
+    report_to = dbid_to_obj(report_to, _PlayerDB)
+
+    # create the correct player entity, using the setup from
+    # base django auth.
+    now = timezone.now()
+    email = typeclass.objects.normalize_email(email)
+    new_player = typeclass(username=key, email=email,
+                           is_staff=is_superuser, is_superuser=is_superuser,
+                           last_login=now, date_joined=now)
+    new_player.set_password(password)
+    new_player._createdict = {"locks":locks, "permissions":permissions,
+                              "report_to":report_to}
+    # saving will trigger the signal that calls the
+    # at_first_save hook on the typeclass, where the _createdict
+    # can be used.
+    new_player.save()
+    return new_player
+
+# alias
+player = create_player

lib/utils/dbserialize.py

@@ -0,0 +1,387 @@
+"""
+This module handles serialization of arbitrary python structural data,
+intended primarily to be stored in the database. It also supports
+storing Django model instances (which plain pickle cannot do).
+
+This serialization is used internally by the server, notably for
+storing data in Attributes and for piping data to process pools.
+
+The purpose of dbserialize is to handle all forms of data. For
+well-structured non-arbitrary exchange, such as communicating with a
+rich web client, a simpler JSON serialization makes more sense.
+
+This module also implements the SaverList, SaverDict and SaverSet
+classes. These are iterables that track their position in a nested
+structure and makes sure to send updates up to their root. This is
+used by Attributes - without it, one would not be able to update mutables
+in-situ, e.g obj.db.mynestedlist[3][5] = 3 would never be saved and
+be out of sync with the database.
+
+"""
+
+from functools import update_wrapper
+from collections import defaultdict, MutableSequence, MutableSet, MutableMapping
+try:
+    from cPickle import dumps, loads
+except ImportError:
+    from pickle import dumps, loads
+from django.db import transaction
+from django.core.exceptions import ObjectDoesNotExist
+from django.contrib.contenttypes.models import ContentType
+from src.server.models import ServerConfig
+from src.utils.utils import to_str, uses_database
+from src.utils import logger
+
+__all__ = ("to_pickle", "from_pickle", "do_pickle", "do_unpickle")
+
+PICKLE_PROTOCOL = 2
+
+# initialization and helpers
+
+_GA = object.__getattribute__
+_SA = object.__setattr__
+_FROM_MODEL_MAP = None
+_TO_MODEL_MAP = None
+_IS_PACKED_DBOBJ = lambda o: type(o) == tuple and len(o) == 4 and o[0] == '__packed_dbobj__'
+if uses_database("mysql") and ServerConfig.objects.get_mysql_db_version() < '5.6.4':
+    # mysql <5.6.4 don't support millisecond precision
+    _DATESTRING = "%Y:%m:%d-%H:%M:%S:000000"
+else:
+    _DATESTRING = "%Y:%m:%d-%H:%M:%S:%f"
+
+
+def _TO_DATESTRING(obj):
+    """
+    this will only be called with valid database objects. Returns datestring
+    on correct form.
+    """
+    try:
+        return _GA(obj, "db_date_created").strftime(_DATESTRING)
+    except AttributeError:
+        # this can happen if object is not yet saved - no datestring is then set
+        obj.save()
+        return _GA(obj, "db_date_created").strftime(_DATESTRING)
+
+
+def _init_globals():
+    "Lazy importing to avoid circular import issues"
+    global _FROM_MODEL_MAP, _TO_MODEL_MAP
+    if not _FROM_MODEL_MAP:
+        _FROM_MODEL_MAP = defaultdict(str)
+        _FROM_MODEL_MAP.update(dict((c.model, c.natural_key()) for c in ContentType.objects.all()))
+    if not _TO_MODEL_MAP:
+        _TO_MODEL_MAP = defaultdict(str)
+        _TO_MODEL_MAP.update(dict((c.natural_key(), c.model_class()) for c in ContentType.objects.all()))
+
+#
+# SaverList, SaverDict, SaverSet - Attribute-specific helper classes and functions
+#
+
+
+def _save(method):
+    "method decorator that saves data to Attribute"
+    def save_wrapper(self, *args, **kwargs):
+        self.__doc__ = method.__doc__
+        ret = method(self, *args, **kwargs)
+        self._save_tree()
+        return ret
+    return update_wrapper(save_wrapper, method)
+
+
+class _SaverMutable(object):
+    """
+    Parent class for properly handling  of nested mutables in
+    an Attribute. If not used something like
+     obj.db.mylist[1][2] = "test" (allocation to a nested list)
+    will not save the updated value to the database.
+    """
+    def __init__(self, *args, **kwargs):
+        "store all properties for tracking the tree"
+        self._parent = kwargs.pop("parent", None)
+        self._db_obj = kwargs.pop("db_obj", None)
+        self._data = None
+
+    def _save_tree(self):
+        "recursively traverse back up the tree, save when we reach the root"
+        if self._parent:
+            self._parent._save_tree()
+        elif self._db_obj:
+            self._db_obj.value = self
+        else:
+            logger.log_errmsg("_SaverMutable %s has no root Attribute to save to." % self)
+
+    def _convert_mutables(self, data):
+        "converts mutables to Saver* variants and assigns .parent property"
+        def process_tree(item, parent):
+            "recursively populate the tree, storing parents"
+            dtype = type(item)
+            if dtype in (basestring, int, long, float, bool, tuple):
+                return item
+            elif dtype == list:
+                dat = _SaverList(parent=parent)
+                dat._data.extend(process_tree(val, dat) for val in item)
+                return dat
+            elif dtype == dict:
+                dat = _SaverDict(parent=parent)
+                dat._data.update((key, process_tree(val, dat)) for key, val in item.items())
+                return dat
+            elif dtype == set:
+                dat = _SaverSet(parent=parent)
+                dat._data.update(process_tree(val, dat) for val in item)
+                return dat
+            return item
+        return process_tree(data, self)
+
+    def __repr__(self):
+        return self._data.__repr__()
+
+    def __len__(self):
+        return self._data.__len__()
+
+    def __iter__(self):
+        return self._data.__iter__()
+
+    def __getitem__(self, key):
+        return self._data.__getitem__(key)
+
+    @_save
+    def __setitem__(self, key, value):
+        self._data.__setitem__(key, self._convert_mutables(value))
+
+    @_save
+    def __delitem__(self, key):
+        self._data.__delitem__(key)
+
+
+class _SaverList(_SaverMutable, MutableSequence):
+    """
+    A list that saves itself to an Attribute when updated.
+    """
+    def __init__(self, *args, **kwargs):
+        super(_SaverList, self).__init__(*args, **kwargs)
+        self._data = list(*args)
+
+    @_save
+    def __add__(self, otherlist):
+        self._data = self._data.__add__(otherlist)
+        return self._data
+
+    @_save
+    def insert(self, index, value):
+        self._data.insert(index, self._convert_mutables(value))
+
+
+class _SaverDict(_SaverMutable, MutableMapping):
+    """
+    A dict that stores changes to an Attribute when updated
+    """
+    def __init__(self, *args, **kwargs):
+        super(_SaverDict, self).__init__(*args, **kwargs)
+        self._data = dict(*args)
+
+    def has_key(self, key):
+        return key in self._data
+
+
+class _SaverSet(_SaverMutable, MutableSet):
+    """
+    A set that saves to an Attribute when updated
+    """
+    def __init__(self, *args, **kwargs):
+        super(_SaverSet, self).__init__(*args, **kwargs)
+        self._data = set(*args)
+
+    def __contains__(self, value):
+        return self._data.__contains__(value)
+
+    @_save
+    def add(self, value):
+        self._data.add(self._convert_mutables(value))
+
+    @_save
+    def discard(self, value):
+        self._data.discard(value)
+
+#
+# serialization helpers
+#
+
+def pack_dbobj(item):
+    """
+    Check and convert django database objects to an internal representation.
+    This either returns the original input item or a tuple
+      ("__packed_dbobj__", key, creation_time, id)
+    """
+    _init_globals()
+    obj = item
+    natural_key = _FROM_MODEL_MAP[hasattr(obj, "id") and hasattr(obj, "db_date_created") and
+                                  hasattr(obj, '__dbclass__') and obj.__dbclass__.__name__.lower()]
+    # build the internal representation as a tuple
+    #  ("__packed_dbobj__", key, creation_time, id)
+    return natural_key and ('__packed_dbobj__', natural_key,
+                             _TO_DATESTRING(obj), _GA(obj, "id")) or item
+
+
+def unpack_dbobj(item):
+    """
+    Check and convert internal representations back to Django database models.
+    The fact that item is a packed dbobj should be checked before this call.
+    This either returns the original input or converts the internal store back
+    to a database representation (its typeclass is returned if applicable).
+    """
+    _init_globals()
+    try:
+        obj = item[3] and _TO_MODEL_MAP[item[1]].objects.get(id=item[3])
+    except ObjectDoesNotExist:
+        return None
+    # even if we got back a match, check the sanity of the date (some
+    # databases may 're-use' the id)
+    return _TO_DATESTRING(obj) == item[2] and obj or None
+
+#
+# Access methods
+#
+
+def to_pickle(data):
+    """
+    This prepares data on arbitrary form to be pickled. It handles any nested
+    structure and returns data on a form that is safe to pickle (including
+    having converted any database models to their internal representation).
+    We also convert any Saver*-type objects back to their normal
+    representations, they are not pickle-safe.
+    """
+    def process_item(item):
+        "Recursive processor and identification of data"
+        dtype = type(item)
+        if dtype in (basestring, int, long, float, bool):
+            return item
+        elif dtype == tuple:
+            return tuple(process_item(val) for val in item)
+        elif dtype in (list, _SaverList):
+            return [process_item(val) for val in item]
+        elif dtype in (dict, _SaverDict):
+            return dict((process_item(key), process_item(val)) for key, val in item.items())
+        elif dtype in (set, _SaverSet):
+            return set(process_item(val) for val in item)
+        elif hasattr(item, '__item__'):
+            # we try to conserve the iterable class, if not convert to list
+            try:
+                return item.__class__([process_item(val) for val in item])
+            except (AttributeError, TypeError):
+                return [process_item(val) for val in item]
+        return pack_dbobj(item)
+    return process_item(data)
+
+
+#@transaction.autocommit
+def from_pickle(data, db_obj=None):
+    """
+    This should be fed a just de-pickled data object. It will be converted back
+    to a form that may contain database objects again. Note that if a database
+    object was removed (or changed in-place) in the database, None will be
+    returned.
+
+    db_obj - this is the model instance (normally an Attribute) that
+             _Saver*-type iterables (_SaverList etc) will save to when they
+             update. It must have a 'value' property that saves assigned data
+             to the database. Skip if not serializing onto a given object.
+
+    If db_obj is given, this function will convert lists, dicts and sets
+    to their _SaverList, _SaverDict and _SaverSet counterparts.
+
+    """
+    def process_item(item):
+        "Recursive processor and identification of data"
+        dtype = type(item)
+        if dtype in (basestring, int, long, float, bool):
+            return item
+        elif _IS_PACKED_DBOBJ(item):
+            # this must be checked before tuple
+            return unpack_dbobj(item)
+        elif dtype == tuple:
+            return tuple(process_item(val) for val in item)
+        elif dtype == dict:
+            return dict((process_item(key), process_item(val)) for key, val in item.items())
+        elif dtype == set:
+            return set(process_item(val) for val in item)
+        elif hasattr(item, '__iter__'):
+            try:
+                # we try to conserve the iterable class if
+                # it accepts an iterator
+                return item.__class__(process_item(val) for val in item)
+            except (AttributeError, TypeError):
+                return [process_item(val) for val in item]
+        return item
+
+    def process_tree(item, parent):
+        "Recursive processor, building a parent-tree from iterable data"
+        dtype = type(item)
+        if dtype in (basestring, int, long, float, bool):
+            return item
+        elif _IS_PACKED_DBOBJ(item):
+            # this must be checked before tuple
+            return unpack_dbobj(item)
+        elif dtype == tuple:
+            return tuple(process_tree(val, item) for val in item)
+        elif dtype == list:
+            dat = _SaverList(parent=parent)
+            dat._data.extend(process_tree(val, dat) for val in item)
+            return dat
+        elif dtype == dict:
+            dat = _SaverDict(parent=parent)
+            dat._data.update(dict((process_item(key), process_tree(val, dat))
+                                   for key, val in item.items()))
+            return dat
+        elif dtype == set:
+            dat = _SaverSet(parent=parent)
+            dat._data.update(set(process_tree(val, dat) for val in item))
+            return dat
+        elif hasattr(item, '__iter__'):
+            try:
+                # we try to conserve the iterable class if it
+                # accepts an iterator
+                return item.__class__(process_tree(val, parent) for val in item)
+            except (AttributeError, TypeError):
+                dat = _SaverList(parent=parent)
+                dat._data.extend(process_tree(val, dat) for val in item)
+                return dat
+        return item
+
+    if db_obj:
+        # convert lists, dicts and sets to their Saved* counterparts. It
+        # is only relevant if the "root" is an iterable of the right type.
+        dtype = type(data)
+        if dtype == list:
+            dat = _SaverList(db_obj=db_obj)
+            dat._data.extend(process_tree(val, parent=dat) for val in data)
+            return dat
+        elif dtype == dict:
+            dat = _SaverDict(db_obj=db_obj)
+            dat._data.update((process_item(key), process_tree(val, parent=dat))
+                              for key, val in data.items())
+            return dat
+        elif dtype == set:
+            dat = _SaverSet(db_obj=db_obj)
+            dat._data.update(process_tree(val, parent=dat) for val in data)
+            return dat
+    return process_item(data)
+
+
+def do_pickle(data):
+    "Perform pickle to string"
+    return to_str(dumps(data, protocol=PICKLE_PROTOCOL))
+
+
+def do_unpickle(data):
+    "Retrieve pickle from pickled string"
+    return loads(to_str(data))
+
+
+def dbserialize(data):
+    "Serialize to pickled form in one step"
+    return do_pickle(to_pickle(data))
+
+
+def dbunserialize(data, db_obj=None):
+    "Un-serialize in one step. See from_pickle for help db_obj."
+    return do_unpickle(from_pickle(data, db_obj=db_obj))

lib/utils/dummyrunner/README.txt

@@ -0,0 +1,6 @@
+
+Dummyrunner
+
+This is a test system for stress-testing the server. It will launch numbers
+of "dummy players" to connect to the server and do various sequences of actions.
+See header of dummyrunner.py for usage.

lib/utils/dummyrunner/dummyrunner.py

@@ -0,0 +1,310 @@
+"""
+Dummy client runner
+
+This module implements a stand-alone launcher for stress-testing
+an Evennia game. It will launch any number of fake clients. These
+clients will log into the server and start doing random operations.
+Customizing and weighing these operations differently depends on
+which type of game is tested. The module contains a testing module
+for plain Evennia.
+
+Please note that you shouldn't run this on a production server!
+Launch the program without any arguments or options to see a
+full step-by-step setup help.
+
+Basically (for testing default Evennia):
+
+ - Use an empty/testing database.
+ - set PERMISSION_PLAYER_DEFAULT = "Builders"
+ - start server, eventually with profiling active
+ - launch this client runner
+
+If you want to customize the runner's client actions
+(because you changed the cmdset or needs to better
+match your use cases or add more actions), you can
+change which actions by adding a path to
+
+   DUMMYRUNNER_ACTIONS_MODULE = <path.to.your.module>
+
+in your settings. See utils.dummyrunner_actions.py
+for instructions on how to define this module.
+
+"""
+
+import os, sys, time, random
+from optparse import OptionParser
+from twisted.conch import telnet
+from twisted.internet import reactor, protocol
+# from twisted.application import internet, service
+# from twisted.web import client
+from twisted.internet.task import LoopingCall
+
+# Tack on the root evennia directory to the python path and initialize django settings
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+os.environ["DJANGO_SETTINGS_MODULE"] = "game.settings"
+#from game import settings
+#try:
+#    from django.conf import settings as settings2
+#    settings2.configure()
+#except RuntimeError:
+#    pass
+#finally:
+#    del settings2
+
+from django.conf import settings
+from src.utils import utils
+
+HELPTEXT = """
+
+Usage: dummyrunner.py [-h][-v][-V] [nclients]
+
+DO NOT RUN THIS ON A PRODUCTION SERVER! USE A CLEAN/TESTING DATABASE!
+
+This stand-alone program launches dummy telnet clients against a
+running Evennia server. The idea is to mimic real players logging in
+and repeatedly doing resource-heavy commands so as to stress test the
+game. It uses the default command set to log in and issue commands, so
+if that was customized, some of the functionality will not be tested
+(it will not fail, the commands will just not be recognized).  The
+running clients will create new objects and rooms all over the place
+as part of their running, so using a clean/testing database is
+strongly recommended.
+
+Setup:
+  1) setup a fresh/clean database (if using sqlite, just safe-copy
+     away your real evennia.db3 file and create a new one with
+     manage.py)
+  2) in game/settings.py, add
+
+        PERMISSION_PLAYER_DEFAULT="Builders"
+
+  3a) Start Evennia like normal.
+  3b) If you want profiling, start Evennia like this instead:
+
+        python runner.py -S start
+
+     this will start Evennia under cProfiler with output server.prof.
+  4) run this dummy runner:
+
+        python dummyclients.py <nr_of_clients> [timestep] [port]
+
+     Default is to connect one client to port 4000, using a 5 second
+     timestep.  Increase the number of clients and shorten the
+     timestep (minimum is 1s) to further stress the game.
+
+     You can stop the dummy runner with Ctrl-C.
+
+  5) Log on and determine if game remains responsive despite the
+     heavier load. Note that if you do profiling, there is an
+     additional overhead from the profiler too!
+  6) If you use profiling, let the game run long enough to gather
+     data, then stop the server. You can inspect the server.prof file
+     from a python prompt (see Python's manual on cProfiler).
+
+"""
+# number of clients to launch if no input is given on command line
+DEFAULT_NCLIENTS = 1
+# time between each 'tick', in seconds, if not set on command
+# line. All launched clients will be called upon to possibly do an
+# action with this frequency.
+DEFAULT_TIMESTEP = 2
+# chance of a client performing an action, per timestep. This helps to
+# spread out usage randomly, like it would be in reality.
+CHANCE_OF_ACTION = 0.05
+# spread out the login action separately, having many players create accounts
+# and connect simultaneously is generally unlikely.
+CHANCE_OF_LOGIN = 0.5
+# Port to use, if not specified on command line
+DEFAULT_PORT = settings.TELNET_PORTS[0]
+#
+NLOGGED_IN = 0
+NCLIENTS = 0
+
+#------------------------------------------------------------
+# Helper functions
+#------------------------------------------------------------
+
+def idcounter():
+    "generates subsequent id numbers"
+    idcount = 0
+    while True:
+        idcount += 1
+        yield idcount
+OID = idcounter()
+CID = idcounter()
+
+def makeiter(obj):
+    "makes everything iterable"
+    if not hasattr(obj, '__iter__'):
+        return [obj]
+    return obj
+
+#------------------------------------------------------------
+# Client classes
+#------------------------------------------------------------
+
+class DummyClient(telnet.StatefulTelnetProtocol):
+    """
+    Handles connection to a running Evennia server,
+    mimicking a real player by sending commands on
+    a timer.
+    """
+
+    def connectionMade(self):
+
+        # public properties
+        self.cid = CID.next()
+        self.istep = 0
+        self.exits = [] # exit names created
+        self.objs = [] # obj names created
+
+        self._report = ""
+        self._cmdlist = [] # already stepping in a cmd definition
+        self._ncmds = 0
+        self._actions = self.factory.actions
+        self._echo_brief = self.factory.verbose == 1
+        self._echo_all = self.factory.verbose == 2
+        #print " ** client %i connected." % self.cid
+
+        reactor.addSystemEventTrigger('before', 'shutdown', self.logout)
+
+        # start client tick
+        d = LoopingCall(self.step)
+        # dissipate exact step by up to +/- 0.5 second
+        timestep = self.factory.timestep + (-0.5 + (random.random()*1.0))
+        d.start(timestep, now=True).addErrback(self.error)
+
+    def dataReceived(self, data):
+        "Echo incoming data to stdout"
+        if self._echo_all:
+            print data
+
+    def connectionLost(self, reason):
+        "loosing the connection"
+        #print " ** client %i lost connection." % self.cid
+
+    def error(self, err):
+        "error callback"
+        print err
+
+    def counter(self):
+        "produces a unique id, also between clients"
+        return OID.next()
+
+    def logout(self):
+        "Causes the client to log out of the server. Triggered by ctrl-c signal."
+        cmd, report = self._actions[1](self)
+        print "client %i %s (%s actions)" % (self.cid, report, self.istep)
+        self.sendLine(cmd)
+
+    def step(self):
+        """
+        Perform a step. This is called repeatedly by the runner
+        and causes the client to issue commands to the server.
+        This holds all "intelligence" of the dummy client.
+        """
+        if self.istep == 0 and random.random() > CHANCE_OF_LOGIN:
+            return
+        elif random.random() > CHANCE_OF_ACTION:
+            return
+
+        global NLOGGED_IN
+        if not self._cmdlist:
+            # no cmdlist in store, get a new one
+            if self.istep == 0:
+                NLOGGED_IN += 1
+                cfunc = self._actions[0]
+            else: # random selection using cumulative probabilities
+                rand = random.random()
+                cfunc = [func for cprob, func in self._actions[2] if cprob >= rand][0]
+            # assign to internal cmdlist
+            cmd, self._report = cfunc(self)
+            self._cmdlist = list(makeiter(cmd))
+            self._ncmds = len(self._cmdlist)
+        # output
+        if self.istep == 0 and not (self._echo_brief or self._echo_all):
+            # only print login
+            print "client %i %s (%i/%i)" % (self.cid, self._report, NLOGGED_IN, NCLIENTS)
+        elif self.istep == 0 or self._echo_brief or self._echo_all:
+            print "client %i %s (%i/%i)" % (self.cid, self._report, self._ncmds-(len(self._cmdlist)-1), self._ncmds)
+        # launch the action by popping the first element from cmdlist (don't hide tracebacks)
+        self.sendLine(str(self._cmdlist.pop(0)))
+        self.istep += 1 # only steps up if an action is taken
+
+class DummyFactory(protocol.ClientFactory):
+    protocol = DummyClient
+    def __init__(self, actions, timestep, verbose):
+        "Setup the factory base (shared by all clients)"
+        self.actions = actions
+        self.timestep = timestep
+        self.verbose = verbose
+
+#------------------------------------------------------------
+# Access method:
+# Starts clients and connects them to a running server.
+#------------------------------------------------------------
+
+def start_all_dummy_clients(actions, nclients=1, timestep=5, telnet_port=4000, verbose=0):
+
+    # validating and preparing the action tuple
+    global NCLIENTS
+    NCLIENTS = nclients
+
+    # make sure the probabilities add up to 1
+    pratio = 1.0 / sum(tup[0] for tup in actions[2:])
+    flogin, flogout, probs, cfuncs = actions[0], actions[1], [tup[0] * pratio for tup in actions[2:]], [tup[1] for tup in actions[2:]]
+    # create cumulative probabilies for the random actions
+    cprobs = [sum(v for i,v in enumerate(probs) if i<=k) for k in range(len(probs))]
+    # rebuild a new, optimized action structure
+    actions = (flogin, flogout, zip(cprobs, cfuncs))
+
+    # setting up all clients (they are automatically started)
+    factory = DummyFactory(actions, timestep, verbose)
+    for i in range(nclients):
+        reactor.connectTCP("localhost", telnet_port, factory)
+    # start reactor
+    reactor.run()
+
+#------------------------------------------------------------
+# Command line interface
+#------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # parsing command line with default vals
+    parser = OptionParser(usage="%prog [options] <nclients> [timestep, [port]]",
+                          description="This program requires some preparations to run properly. Start it without any arguments or options for full help.")
+    parser.add_option('-v', '--verbose', action='store_const', const=1, dest='verbose',
+                      default=0,help="echo brief description of what clients do every timestep.")
+    parser.add_option('-V', '--very-verbose', action='store_const',const=2, dest='verbose',
+                      default=0,help="echo all client returns to stdout (hint: use only with nclients=1!)")
+
+    options, args = parser.parse_args()
+
+    nargs = len(args)
+    nclients = DEFAULT_NCLIENTS
+    timestep = DEFAULT_TIMESTEP
+    port = DEFAULT_PORT
+    try:
+        if not args : raise Exception
+        if nargs > 0: nclients = max(1, int(args[0]))
+        if nargs > 1: timestep = max(1, int(args[1]))
+        if nargs > 2: port = int(args[2])
+    except Exception:
+        print HELPTEXT
+        sys.exit()
+
+    # import the ACTION tuple from a given module
+    try:
+        action_modpath = settings.DUMMYRUNNER_ACTIONS_MODULE
+    except AttributeError:
+        # use default
+        action_modpath = "src.utils.dummyrunner.dummyrunner_actions"
+    actions = utils.variable_from_module(action_modpath, "ACTIONS")
+
+    print "Connecting %i dummy client(s) to port %i using a %i second timestep ... " % (nclients, port, timestep)
+    t0 = time.time()
+    start_all_dummy_clients(actions, nclients, timestep, port,
+                       verbose=options.verbose)
+    ttot = time.time() - t0
+    print "... dummy client runner finished after %i seconds." % ttot

lib/utils/dummyrunner/dummyrunner_actions.py

@@ -0,0 +1,231 @@
+"""
+These are actions for the dummy client runner, using
+the default command set and intended for unmodified Evennia.
+
+Each client action is defined as a function. The clients
+will perform these actions randomly (except the login action).
+
+Each action-definition function should take one argument- "client",
+which is a reference to the client currently performing the action
+Use the client object for saving data between actions.
+
+The client object has the following relevant properties and methods:
+  cid - unique client id
+  istep - the current step
+  exits - an empty list. Can be used to store exit names
+  objs - an empty list. Can be used to store object names
+  counter() - get an integer value. This counts up for every call and
+              is always unique between clients.
+
+The action-definition function should return the command that the
+client should send to the server (as if it was input in a mud client).
+It should also return a string detailing the action taken. This string is
+used by the "brief verbose" mode of the runner and is prepended by
+"Client N " to produce output like "Client 3 is creating objects ..."
+
+This module *must* also define a variable named ACTIONS. This is a tuple
+where the first element is the function object for the action function
+to call when the client logs onto the server.  The following elements
+are 2-tuples (probability, action_func), where probability defines how
+common it is for that particular action to happen. The runner will
+randomly pick between those functions based on the probability.
+
+ACTIONS = (login_func, (0.3, func1), (0.1, func2) ... )
+
+To change the runner to use your custom ACTION and/or action
+definitions, edit settings.py and add
+
+ DUMMYRUNNER_ACTIONS_MODULE = "path.to.your.module"
+
+"""
+
+# it's very useful to have a unique id for this run to avoid any risk
+# of clashes
+
+import time
+RUNID = time.time()
+
+# some convenient templates
+
+START_ROOM = "testing_room_start-%s-%s" % (RUNID, "%i")
+ROOM_TEMPLATE = "testing_room_%s-%s" % (RUNID, "%i")
+EXIT_TEMPLATE = "exit_%s-%s" % (RUNID, "%i")
+OBJ_TEMPLATE = "testing_obj_%s-%s" % (RUNID, "%i")
+TOBJ_TEMPLATE = "testing_button_%s-%s" % (RUNID, "%i")
+TOBJ_TYPECLASS = "examples.red_button.RedButton"
+
+# action function definitions
+
+def c_login(client):
+    "logins to the game"
+    cname = "Dummy-%s-%i" % (RUNID, client.cid)
+    #cemail = "%s@dummy.com" % (cname.lower())
+    cpwd = "%s-%s" % (RUNID, client.cid)
+    # set up for digging a first room (to move to)
+    roomname = ROOM_TEMPLATE % client.counter()
+    exitname1 = EXIT_TEMPLATE % client.counter()
+    exitname2 = EXIT_TEMPLATE % client.counter()
+    client.exits.extend([exitname1, exitname2])
+    #cmd = '@dig %s = %s, %s' % (roomname, exitname1, exitname2)
+    cmd = ('create %s %s' % (cname, cpwd),
+           'connect %s %s' % (cname, cpwd),
+           '@dig %s' % START_ROOM % client.cid,
+           '@teleport %s' % START_ROOM % client.cid,
+           '@dig %s = %s, %s' % (roomname, exitname1, exitname2)
+           )
+
+    return cmd, "logs in as %s ..." % cname
+
+def c_login_nodig(client):
+    "logins, don't dig its own room"
+    cname = "Dummy-%s-%i" % (RUNID, client.cid)
+    cpwd = "%s-%s" % (RUNID, client.cid)
+    cmd = ('create %s %s' % (cname, cpwd),
+           'connect %s %s' % (cname, cpwd))
+    return cmd, "logs in as %s ..." % cname
+
+def c_logout(client):
+    "logouts of the game"
+    return "@quit", "logs out"
+
+def c_looks(client):
+    "looks at various objects"
+    cmd = ["look %s" % obj for obj in client.objs]
+    if not cmd:
+        cmd = ["look %s" % exi for exi in client.exits]
+        if not cmd:
+            cmd = "look"
+    return cmd, "looks ..."
+
+def c_examines(client):
+    "examines various objects"
+    cmd = ["examine %s" % obj for obj in client.objs]
+    if not cmd:
+        cmd = ["examine %s" % exi for exi in client.exits]
+    if not cmd:
+        cmd = "examine me"
+    return cmd, "examines objs ..."
+
+def c_help(client):
+    "reads help files"
+    cmd = ('help',
+           'help @teleport',
+           'help look',
+           'help @tunnel',
+           'help @dig')
+    return cmd, "reads help ..."
+
+def c_digs(client):
+    "digs a new room, storing exit names on client"
+    roomname = ROOM_TEMPLATE % client.counter()
+    exitname1 = EXIT_TEMPLATE % client.counter()
+    exitname2 = EXIT_TEMPLATE % client.counter()
+    client.exits.extend([exitname1, exitname2])
+    cmd = '@dig/tel %s = %s, %s' % (roomname, exitname1, exitname2)
+    return cmd, "digs ..."
+
+def c_creates_obj(client):
+    "creates normal objects, storing their name on client"
+    objname = OBJ_TEMPLATE % client.counter()
+    client.objs.append(objname)
+    cmd = ('@create %s' % objname,
+           '@desc %s = "this is a test object' % objname,
+           '@set %s/testattr = this is a test attribute value.' % objname,
+           '@set %s/testattr2 = this is a second test attribute.' % objname)
+    return cmd, "creates obj ..."
+
+def c_creates_button(client):
+    "creates example button, storing name on client"
+    objname = TOBJ_TEMPLATE % client.counter()
+    client.objs.append(objname)
+    cmd = ('@create %s:%s' % (objname, TOBJ_TYPECLASS),
+           '@desc %s = test red button!' % objname)
+    return cmd, "creates button ..."
+
+def c_socialize(client):
+    "socializechats on channel"
+    cmd = ('ooc Hello!',
+          'ooc Testing ...',
+          'ooc Testing ... times 2',
+          'say Yo!',
+          'emote stands looking around.')
+    return cmd, "socializes ..."
+
+def c_moves(client):
+    "moves to a previously created room, using the stored exits"
+    cmd = client.exits # try all exits - finally one will work
+    if not cmd: cmd = "look"
+    return cmd, "moves ..."
+
+def c_moves_n(client):
+    "move through north exit if available"
+    cmd = ("north",)
+    return cmd, "moves n..."
+
+def c_moves_s(client):
+    "move through north exit if available"
+    cmd = ("north",)
+    return cmd, "moves s..."
+
+# Action tuple (required)
+#
+# This is a tuple of client action functions. The first element is the
+# function the client should use to log into the game and move to
+# STARTROOM . The second element is the logout command, for cleanly
+# exiting the mud. The following elements are 2-tuples of (probability,
+# action_function). The probablities should normally sum up to 1,
+# otherwise the system will normalize them.
+#
+
+## "normal builder" definitionj
+#ACTIONS = ( c_login,
+#            c_logout,
+#            (0.5, c_looks),
+#            (0.08, c_examines),
+#            (0.1, c_help),
+#            (0.01, c_digs),
+#            (0.01, c_creates_obj),
+#            (0.3, c_moves))
+## "heavy" builder definition
+#ACTIONS = ( c_login,
+#            c_logout,
+#            (0.2, c_looks),
+#            (0.1, c_examines),
+#            (0.2, c_help),
+#            (0.1, c_digs),
+#            (0.1, c_creates_obj),
+#            #(0.01, c_creates_button),
+#            (0.2, c_moves))
+## "passive player" definition
+#ACTIONS = ( c_login,
+#            c_logout,
+#            (0.7, c_looks),
+#            #(0.1, c_examines),
+#            (0.3, c_help))
+#            #(0.1, c_digs),
+#            #(0.1, c_creates_obj),
+#            #(0.1, c_creates_button),
+#            #(0.4, c_moves))
+## "normal player" definition
+ACTIONS = ( c_login,
+            c_logout,
+            (0.01, c_digs),
+            (0.39, c_looks),
+            (0.2, c_help),
+            (0.4, c_moves))
+#ACTIONS = (c_login_nodig,
+#           c_logout,
+#           (1.0, c_moves_n))
+## "socializing heavy builder" definition
+#ACTIONS = (c_login,
+#           c_logout,
+#           (0.1, c_socialize),
+#           (0.1, c_looks),
+#           (0.2, c_help),
+#           (0.1, c_creates_obj),
+#           (0.2, c_digs),
+#           (0.3, c_moves))
+## "heavy digger memory tester" definition
+#ACTIONS = (c_login,
+#           c_logout,
+#           (1.0, c_digs))

lib/utils/dummyrunner/memplot.py

@@ -0,0 +1,100 @@
+"""
+Script that saves memory and idmapper data over time.
+
+Data will be saved to game/logs/memoryusage.log. Note that
+the script will append to this file if it already exists.
+
+Call this module directly to plot the log (requires matplotlib and numpy).
+"""
+import os, sys
+import time
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'game.settings'
+import ev
+from src.utils.idmapper import base as _idmapper
+
+LOGFILE = "logs/memoryusage.log"
+INTERVAL = 30 # log every 30 seconds
+
+class Memplot(ev.Script):
+    def at_script_creation(self):
+        self.key = "memplot"
+        self.desc = "Save server memory stats to file"
+        self.start_delay = False
+        self.persistent = True
+        self.interval = INTERVAL
+        self.db.starttime = time.time()
+
+    def at_repeat(self):
+
+        pid = os.getpid()
+        rmem = float(os.popen('ps -p %d -o %s | tail -1' % (pid, "rss")).read()) / 1000.0  # resident memory
+        vmem = float(os.popen('ps -p %d -o %s | tail -1' % (pid, "vsz")).read()) / 1000.0  # virtual memory
+        total_num, cachedict = _idmapper.cache_size()
+        t0 = (time.time() - self.db.starttime) / 60.0 # save in minutes
+
+        with open(LOGFILE, "a") as f:
+            f.write("%s, %s, %s, %s\n" % (t0, rmem, vmem, int(total_num)))
+
+if __name__ == "__main__":
+
+    # plot output from the file
+
+    from matplotlib import pyplot as pp
+    import numpy
+
+    data = numpy.genfromtxt("../../../game/" + LOGFILE, delimiter=",")
+    secs = data[:,0]
+    rmem = data[:,1]
+    vmem = data[:,2]
+    nobj = data[:,3]
+
+    # calculate derivative of obj creation
+    #oderiv = (0.5*(nobj[2:] - nobj[:-2]) / (secs[2:] - secs[:-2])).copy()
+    #oderiv = (0.5*(rmem[2:] - rmem[:-2]) / (secs[2:] - secs[:-2])).copy()
+
+    fig = pp.figure()
+    ax1 = fig.add_subplot(111)
+    ax1.set_title("1000 bots (normal players with light building)")
+    ax1.set_xlabel("Time (mins)")
+    ax1.set_ylabel("Memory usage (MB)")
+    ax1.plot(secs, rmem, "r", label="RMEM", lw=2)
+    ax1.plot(secs, vmem, "b", label="VMEM", lw=2)
+    ax1.legend(loc="upper left")
+
+    ax2 = ax1.twinx()
+    ax2.plot(secs, nobj, "g--", label="objs in cache", lw=2)
+    #ax2.plot(secs[:-2], oderiv/60.0, "g--", label="Objs/second", lw=2)
+    #ax2.plot(secs[:-2], oderiv, "g--", label="Objs/second", lw=2)
+    ax2.set_ylabel("Number of objects")
+    ax2.legend(loc="lower right")
+    ax2.annotate("First 500 bots\nconnecting", xy=(10, 4000))
+    ax2.annotate("Next 500 bots\nconnecting", xy=(350,10000))
+    #ax2.annotate("@reload", xy=(185,600))
+
+#    # plot mem vs cachesize
+#    nobj, rmem, vmem = nobj[:262].copy(), rmem[:262].copy(), vmem[:262].copy()
+#
+#    fig = pp.figure()
+#    ax1 = fig.add_subplot(111)
+#    ax1.set_title("Memory usage per cache size")
+#    ax1.set_xlabel("Cache size (number of objects)")
+#    ax1.set_ylabel("Memory usage (MB)")
+#    ax1.plot(nobj, rmem, "r", label="RMEM", lw=2)
+#    ax1.plot(nobj, vmem, "b", label="VMEM", lw=2)
+#
+
+##    # empirical estimate of memory usage: rmem = 35.0 + 0.0157 * Ncache
+##    # Ncache = int((rmem - 35.0) / 0.0157)  (rmem in MB)
+#
+#    rderiv_aver = 0.0157
+#    fig = pp.figure()
+#    ax1 = fig.add_subplot(111)
+#    ax1.set_title("Relation between memory and cache size")
+#    ax1.set_xlabel("Memory usage (MB)")
+#    ax1.set_ylabel("Idmapper Cache Size (number of objects)")
+#    rmem = numpy.linspace(35, 2000, 2000)
+#    nobjs = numpy.array([int((mem - 35.0) / 0.0157) for mem in rmem])
+#    ax1.plot(rmem, nobjs, "r", lw=2)
+
+    pp.show()

lib/utils/dummyrunner/test_queries.py

@@ -0,0 +1,41 @@
+"""
+This is a little routine for viewing the sql queries that are executed by a given
+query as well as count them for optimization testing.
+
+"""
+import sys, os
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+os.environ["DJANGO_SETTINGS_MODULE"] = "game.settings"
+from django.db import connection
+
+
+def count_queries(exec_string, setup_string):
+    """
+    Display queries done by exec_string. Use setup_string
+    to setup the environment to test.
+    """
+
+    exec setup_string
+
+    num_queries_old = len(connection.queries)
+    exec exec_string
+    nqueries = len(connection.queries) - num_queries_old
+
+    for query in connection.queries[-nqueries if nqueries else 1:]:
+        print query["time"], query["sql"]
+    print "Number of queries: %s" % nqueries
+
+if __name__ == "__main__":
+
+    # setup tests here
+
+    setup_string = \
+"""
+from src.objects.models import ObjectDB
+g = ObjectDB.objects.get(db_key="Griatch")
+"""
+    exec_string = \
+"""
+g.tags.all()
+"""
+    count_queries(exec_string, setup_string)

lib/utils/evennia-mode.el

@@ -0,0 +1,81 @@
+;
+; Emacs major mode for editing Evennia batch-command files (*.ev files). 
+; Griatch 2011-09. Tested with GNU Emacs 23. Released under same license as Evennia.
+;
+; For batch-code files it's better to simply use the normal Python mode. 
+;
+; Features: 
+;  Syntax hilighting
+;  Auto-indenting properly when pressing <tab>. 
+; 
+; Installation: 
+;   - Copy this file, evennia-mode.el, to a location where emacs looks for plugins
+;     (usually .emacs.d/ at least under Linux)
+;   - If you don't have that directory, either look on the web for how to find it 
+;     or create it yourself - create a new directory .emacs.d/ some place and add 
+;     the following to emacs' configuration file (.emacs):
+;        (add-to-list 'load-path "<PATH>/.emacs.d/")
+;     where PATH is the place you created the directory. Now Emacs will know to
+;     look here for plugins. Copy this file there.
+;   - In emacs config file (.emacs), next add the following line:
+;        (require 'evennia-mode)    
+;   - (re)start emacs 
+;   - Open a batch file with the ending *.ev. The mode will start automatically
+;     (otherwise you can manually start it with M-x evennia-mode).     
+;
+;   Report bugs to evennia's issue tracker.
+;
+
+(defvar evennia-mode-hook nil)
+
+; Add keyboard shortcuts (not used)
+(defvar evennia-mode-map
+  (let ((map (make-sparse-keymap)))
+    (define-key map "\C-j" 'newline-and-indent)
+    map)
+  "Keymap for evennia major mode")
+
+; Autoload this when .ev file opens.
+(add-to-list 'auto-mode-alist '("\\.ev\\'" . evennia-mode))
+
+; Syntax hilighting
+(defconst evennia-font-lock-keywords
+  (list
+   '("^ *#.*" . font-lock-comment-face)
+   '("^[^ |^#]*" . font-lock-variable-name-face))
+  ;'("^[^ #].*" . font-lock-variable-name-face)) ; more extreme hilight
+  "Minimal highlighting for evennia ev files."
+  )  
+
+; Auto-indentation 
+(defun evennia-indent-line ()
+  "Indent current line as batch-code"
+  (interactive)
+  (beginning-of-line)
+  (if (looking-at "^ *#") ; a comment line
+      (indent-line-to 0)
+    (progn
+      (forward-line -1) ; back up one line 
+      (if (looking-at "^ *#") ; previous line was comment
+          (progn
+            (forward-line)
+            (indent-line-to 0))
+        (progn
+          (forward-line)
+          (indent-line-to 1)))))
+  )
+
+; Register with Emacs system 
+(defun evennia-mode ()
+  "Major mode for editing Evennia batch-command files."
+  (interactive)
+  (kill-all-local-variables)
+  (use-local-map evennia-mode-map)
+  (set (make-local-variable 'indent-line-function) 'evennia-indent-line)
+  (set (make-local-variable 'font-lock-defaults) '(evennia-font-lock-keywords))
+  (setq major-mode 'evennia-mode)
+  (setq mode-name "evennia")
+  (run-hooks 'evennia-mode-hook)
+)
+
+(provide 'evennia-mode)

lib/utils/evform.py

@@ -0,0 +1,439 @@
+# coding=utf-8
+"""
+EvForm - a way to create advanced ascii forms
+
+This is intended for creating advanced ascii game forms, such as a
+large pretty character sheet or info document.
+
+The system works on the basis of a readin template that is given in a
+separate python file imported into the handler. This file contains
+some optional settings and a string mapping out the form. The template
+has markers in it to denounce fields to fill. The markers map the
+absolute size of the field and will be filled with an evtable.EvCell
+object when displaying the form.
+
+Note, when printing examples with ANSI color, you need to wrap
+the output in unicode(), such as print unicode(form). This is
+due to a bug in the Python parser and the print statement.
+
+
+Example of input file testform.py:
+
+FORMCHAR = "x"
+TABLECHAR = "c"
+
+FORM = '''
+.------------------------------------------------.
+|                                                |
+|  Name: xxxxx1xxxxx    Player: xxxxxxx2xxxxxxx  |
+|        xxxxxxxxxxx                             |
+|                                                |
+ >----------------------------------------------<
+|                                                |
+| Desc:  xxxxxxxxxxx    STR: x4x    DEX: x5x     |
+|        xxxxx3xxxxx    INT: x6x    STA: x7x     |
+|        xxxxxxxxxxx    LUC: x8x    MAG: x9x     |
+|                                                |
+ >----------------------------------------------<
+|          |                                     |
+| cccccccc | ccccccccccccccccccccccccccccccccccc |
+| cccccccc | ccccccccccccccccccccccccccccccccccc |
+| cccAcccc | ccccccccccccccccccccccccccccccccccc |
+| cccccccc | ccccccccccccccccccccccccccccccccccc |
+| cccccccc | cccccccccccccccccBccccccccccccccccc |
+|          |                                     |
+-------------------------------------------------
+'''
+
+The first line of the FORM string is ignored. The forms and table
+markers must mark out complete, unbroken rectangles, each containing
+one embedded single-character identifier (so the smallest element
+possible is a 3-character wide form). The identifier can be any
+character except for the FORM_CHAR and TABLE_CHAR and some of the
+common ascii-art elements, like space, _ | * etc (see
+INVALID_FORMCHARS in this module). Form Rectangles can have any size,
+but must be separated from each other by at least one other
+character's width.
+
+
+Use as follows:
+
+    import evform
+
+    # create a new form from the template
+    form = evform.EvForm("path/to/testform.py")
+
+    (MudForm can also take a dictionary holding
+     the required keys FORMCHAR, TABLECHAR and FORM)
+
+    # add data to each tagged form cell
+    form.map(cells={1: "Tom the Bouncer",
+                    2: "Griatch",
+                    3: "A sturdy fellow",
+                    4: 12,
+                    5: 10,
+                    6:  5,
+                    7: 18,
+                    8: 10,
+                    9:  3})
+    # create the EvTables
+    tableA = evform.EvTable("HP","MV","MP",
+                               table=[["**"], ["*****"], ["***"]],
+                               border="incols")
+    tableB = evform.EvTable("Skill", "Value", "Exp",
+                               table=[["Shooting", "Herbalism", "Smithing"],
+                                      [12,14,9],["550/1200", "990/1400", "205/900"]],
+                               border="incols")
+    # add the tables to the proper ids in the form
+    form.map(tables={"A": tableA,
+                     "B": tableB}
+
+    # unicode is required since the example contains non-ascii characters
+    print unicode(form)
+
+This produces the following result:
+
+.------------------------------------------------.
+|                                                |
+|  Name: Tom the        Player: Griatch          |
+|        Bouncer                                 |
+|                                                |
+ >----------------------------------------------<
+|                                                |
+| Desc:  A sturdy       STR: 12     DEX: 10      |
+|        fellow         INT: 5      STA: 18      |
+|                       LUC: 10     MAG: 3       |
+|                                                |
+ >----------------------------------------------<
+|          |                                     |
+| HP|MV|MP | Skill      |Value      |Exp         |
+| ~~+~~+~~ | ~~~~~~~~~~~+~~~~~~~~~~~+~~~~~~~~~~~ |
+| **|**|** | Shooting   |12         |550/1200    |
+|   |**|*  | Herbalism  |14         |990/1400    |
+|   |* |   | Smithing   |9          |205/900     |
+|          |                                     |
+ ------------------------------------------------
+
+The marked forms have been replaced with EvCells of text and with
+EvTables. The form can be updated by simply re-applying form.map()
+with the updated data.
+
+When working with the template ascii file, you can use form.reload()
+to re-read the template and re-apply all existing mappings.
+
+Each component is restrained to the width and height specified by the
+template, so it will resize to fit (or crop text if the area is too
+small for it. If you try to fit a table into an area it cannot fit
+into (when including its borders and at least one line of text), the
+form will raise an error.
+
+"""
+
+import re
+import copy
+from src.utils.evtable import EvCell, EvTable
+from src.utils.utils import all_from_module, to_str, to_unicode
+from src.utils.ansi import ANSIString
+
+# non-valid form-identifying characters (which can thus be
+# used as separators between forms without being detected
+# as an identifier). These should be listed in regex form.
+
+INVALID_FORMCHARS = r"\s\/\|\\\*\_\-\#\<\>\~\^\:\;\.\,"
+
+def _to_ansi(obj, regexable=False):
+    "convert to ANSIString"
+    if isinstance(obj, dict):
+        return dict((key, _to_ansi(value, regexable=regexable)) for key, value in obj.items())
+    elif hasattr(obj, "__iter__"):
+        return [_to_ansi(o) for o in obj]
+    else:
+        return ANSIString(to_unicode(obj), regexable=regexable)
+
+class EvForm(object):
+    """
+    This object is instantiated with a text file and parses
+    it for rectangular form fields. It can then be fed a
+    mapping so as to populate the fields with fixed-width
+    EvCell or Tablets.
+
+    """
+    def __init__(self, filename=None, cells=None, tables=None, form=None, **kwargs):
+        """
+        Initiate the form
+
+        keywords:
+            filename - path to template file
+            form - dictionary of {"CELLCHAR":char,
+                                  "TABLECHAR":char,
+                                  "FORM":templatestring}
+                    if this is given, filename is not read.
+            cells - a dictionary mapping of {id:text}
+            tables -  dictionary mapping of {id:EvTable}
+
+        other kwargs are fed as options to the EvCells and EvTables
+        (see evtablet.EvCell and evtable.EvTable for more info).
+
+        """
+        self.filename = filename
+        self.input_form_dict = form
+
+        self.cells_mapping =  dict((to_str(key, force_string=True), value) for key, value in cells.items()) if cells  else {}
+        self.tables_mapping = dict((to_str(key, force_string=True), value) for key, value in tables.items()) if tables else {}
+
+        self.cellchar = "x"
+        self.tablechar = "c"
+
+        self.raw_form = []
+        self.form = []
+
+        # clean kwargs (these cannot be overridden)
+        kwargs.pop("enforce_size", None)
+        kwargs.pop("width", None)
+        kwargs.pop("height", None)
+        # table/cell options
+        self.options = kwargs
+
+        self.reload()
+
+    def _parse_rectangles(self, cellchar, tablechar, form, **kwargs):
+        """
+        Parse a form for rectangular formfields identified by
+        formchar enclosing an identifier.
+        """
+
+        # update options given at creation with new input - this
+        # allows e.g. self.map() to add custom settings for individual
+        # cells/tables
+        custom_options = copy.copy(self.options)
+        custom_options.update(kwargs)
+
+        nform = len(form)
+
+        mapping = {}
+        cell_coords = {}
+        table_coords = {}
+
+        # Locate the identifier tags and the horizontal end coords for all forms
+        re_cellchar =  re.compile(r"%s+([^%s%s])%s+" % (cellchar, INVALID_FORMCHARS, cellchar, cellchar))
+        re_tablechar = re.compile(r"%s+([^%s%s|])%s+" % (tablechar, INVALID_FORMCHARS, tablechar, tablechar))
+        for iy, line in enumerate(_to_ansi(form, regexable=True)):
+            # find cells
+            ix0 = 0
+            while True:
+                match = re_cellchar.search(line, ix0)
+                if match:
+                    # get the width of the rectangle directly from the match
+                    cell_coords[match.group(1)] = [iy, match.start(), match.end()]
+                    ix0 = match.end()
+                else:
+                    break
+            # find tables
+            ix0 = 0
+            while True:
+                match = re_tablechar.search(line, ix0)
+                if match:
+                    # get the width of the rectangle directly from the match
+                    table_coords[match.group(1)] = [iy, match.start(), match.end()]
+                    ix0 = match.end()
+                else:
+                    break
+        #print "cell_coords:", cell_coords
+        #print "table_coords:", table_coords
+
+        # get rectangles and assign EvCells
+        for key, (iy, leftix, rightix) in cell_coords.items():
+
+            # scan up to find top of rectangle
+            dy_up = 0
+            if iy > 0:
+                for i in range(1,iy):
+                    #print "dy_up:", [form[iy-i][ix] for ix in range(leftix, rightix)]
+                    if all(form[iy-i][ix] == cellchar for ix in range(leftix, rightix)):
+                        dy_up += 1
+                    else:
+                        break
+            # find bottom edge of rectangle
+            dy_down = 0
+            if iy < nform-1:
+                for i in range(1,nform-iy-1):
+                    #print "dy_down:", [form[iy+i][ix]for ix in range(leftix, rightix)]
+                    if all(form[iy+i][ix] == cellchar for ix in range(leftix, rightix)):
+                        dy_down += 1
+                    else:
+                        break
+
+            #  we have our rectangle. Calculate size of EvCell.
+            iyup = iy - dy_up
+            iydown = iy + dy_down
+            width = rightix - leftix
+            height = abs(iyup - iydown) + 1
+
+            # we have all the coordinates we need. Create EvCell.
+            data = self.cells_mapping.get(key, "")
+            #if key == "1":
+            #    print "creating cell '%s' (%s):" % (key, data)
+            #    print "iy=%s, iyup=%s, iydown=%s, leftix=%s, rightix=%s, width=%s, height=%s" % (iy, iyup, iydown, leftix, rightix, width, height)
+
+            options = { "pad_left":0, "pad_right":0, "pad_top":0, "pad_bottom":0, "align":"l", "valign":"t", "enforce_size":True}
+            options.update(custom_options)
+            #if key=="4":
+            #print "options:", options
+
+            mapping[key] = (iyup, leftix, width, height, EvCell(data, width=width, height=height,**options))
+
+        # get rectangles and assign Tables
+        for key, (iy, leftix, rightix) in table_coords.items():
+
+            # scan up to find top of rectangle
+            dy_up = 0
+            if iy > 0:
+                for i in range(1,iy):
+                    #print "dy_up:", [form[iy-i][ix] for ix in range(leftix, rightix)]
+                    if all(form[iy-i][ix] == tablechar for ix in range(leftix, rightix)):
+                        dy_up += 1
+                    else:
+                        break
+            # find bottom edge of rectangle
+            dy_down = 0
+            if iy < nform-1:
+                for i in range(1,nform-iy-1):
+                    #print "dy_down:", [form[iy+i][ix]for ix in range(leftix, rightix)]
+                    if all(form[iy+i][ix] == tablechar for ix in range(leftix, rightix)):
+                        dy_down += 1
+                    else:
+                        break
+
+            #  we have our rectangle. Calculate size of Table.
+            iyup = iy - dy_up
+            iydown = iy + dy_down
+            width = rightix - leftix
+            height = abs(iyup - iydown) + 1
+
+            # we have all the coordinates we need. Create Table.
+            table = self.tables_mapping.get(key, None)
+            #print "creating table '%s' (%s):" % (key, data)
+            #print "iy=%s, iyup=%s, iydown=%s, leftix=%s, rightix=%s, width=%s, height=%s" % (iy, iyup, iydown, leftix, rightix, width, height)
+
+            options = { "pad_left":0, "pad_right":0, "pad_top":0, "pad_bottom":0,
+                        "align":"l", "valign":"t", "enforce_size":True}
+            options.update(custom_options)
+            #print "options:", options
+
+            if table:
+                table.reformat(width=width, height=height, **options)
+            else:
+                table = EvTable(width=width, height=height, **options)
+            mapping[key] = (iyup, leftix, width, height, table)
+
+        return mapping
+
+    def _populate_form(self, raw_form, mapping):
+        """
+        Insert cell contents into form at given locations
+        """
+        form = copy.copy(raw_form)
+        for key, (iy0, ix0, width, height, cell_or_table) in mapping.items():
+            # rect is a list of <height> lines, each <width> wide
+            rect = cell_or_table.get()
+            for il, rectline in enumerate(rect):
+                formline = form[iy0+il]
+                # insert new content, replacing old
+                form[iy0+il] = formline = formline[:ix0] + rectline + formline[ix0+width:]
+        return form
+
+    def map(self, cells=None, tables=None, **kwargs):
+        """
+        Add mapping for form.
+
+        cells - a dictionary of {identifier:celltext}
+        tables - a dictionary of {identifier:table}
+
+        kwargs will be forwarded to tables/cells. See
+        evtable.EvCell and evtable.EvTable for info.
+
+        """
+        # clean kwargs (these cannot be overridden)
+        kwargs.pop("enforce_size", None)
+        kwargs.pop("width", None)
+        kwargs.pop("height", None)
+
+        new_cells =  dict((to_str(key, force_string=True), value) for key, value in cells.items()) if cells  else {}
+        new_tables = dict((to_str(key, force_string=True), value) for key, value in tables.items()) if tables else {}
+
+        self.cells_mapping.update(new_cells)
+        self.tables_mapping.update(new_tables)
+        self.reload()
+
+    def reload(self, filename=None, form=None, **kwargs):
+        """
+        Creates the form from a stored file name
+        """
+        # clean kwargs (these cannot be overridden)
+        kwargs.pop("enforce_size", None)
+        kwargs.pop("width", None)
+        kwargs.pop("height", None)
+
+        if form or self.input_form_dict:
+            datadict = form if form else self.input_form_dict
+            self.input_form_dict = datadict
+        elif filename or self.filename:
+            filename = filename if filename else self.filename
+            datadict = all_from_module(filename)
+            self.filename = filename
+        else:
+            datadict = {}
+
+        cellchar = to_str(datadict.get("FORMCHAR", "x"))
+        self.cellchar = to_str(cellchar[0] if len(cellchar) > 1 else cellchar)
+        tablechar = datadict.get("TABLECHAR", "c")
+        self.tablechar = tablechar[0] if len(tablechar) > 1 else tablechar
+
+        # split into a list of list of lines. Form can be indexed with form[iy][ix]
+        self.raw_form = _to_ansi(to_unicode(datadict.get("FORM", "")).split("\n"))
+        # strip first line
+        self.raw_form = self.raw_form[1:] if self.raw_form else self.raw_form
+
+        self.options.update(kwargs)
+
+        # parse and replace
+        self.mapping = self._parse_rectangles(self.cellchar, self.tablechar, self.raw_form, **kwargs)
+        self.form = self._populate_form(self.raw_form, self.mapping)
+
+    def __str__(self):
+        "Prints the form"
+        return ANSIString("\n").join([line for line in self.form])
+
+    def __unicode__(self):
+        "prints the form"
+        return unicode(ANSIString("\n").join([line for line in self.form]))
+
+def _test():
+    "test evform"
+    form = EvForm("src.utils.evform_test")
+
+    # add data to each tagged form cell
+    form.map(cells={1: "{gTom the Bouncer{n",
+                    2: "{yGriatch{n",
+                    3: "A sturdy fellow",
+                    4: 12,
+                    5: 10,
+                    6:  5,
+                    7: 18,
+                    8: 10,
+                    9:  3})
+    # create the EvTables
+    tableA = EvTable("HP","MV","MP",
+                               table=[["**"], ["*****"], ["***"]],
+                               border="incols")
+    tableB = EvTable("Skill", "Value", "Exp",
+                               table=[["Shooting", "Herbalism", "Smithing"],
+                                      [12,14,9],["550/1200", "990/1400", "205/900"]],
+                               border="incols")
+    # add the tables to the proper ids in the form
+    form.map(tables={"A": tableA,
+                     "B": tableB})
+
+    # unicode is required since the example contains non-ascii characters
+    print unicode(form)
+    return form

lib/utils/evform_test.py

@@ -0,0 +1,32 @@
+# encoding=utf-8
+"""
+Test form
+"""
+
+FORMCHAR = "x"
+TABLECHAR = "c"
+
+FORM = """
+.------------------------------------------------.
+|                                                |
+|  Name: xxxxx1xxxxx    Player: xxxxxxx2xxxxxxx  |
+|        xxxxxxxxxxx                             |
+|                                                |
+ >----------------------------------------------<
+|                                                |
+| Desc:  xxxxxxxxxxx    STR: x4x    DEX: x5x     |
+|        xxxxx3xxxxx    INT: x6x    STA: x7x     |
+|        xxxxxxxxxxx    LUC: x8x    MAG: x9x     |
+|                                                |
+ >----------------------------------------------<
+|          |                                     |
+| cccccccc | ccccccccccccccccccccccccccccccccccc |
+| cccccccc | ccccccccccccccccccccccccccccccccccc |
+| cccAcccc | ccccccccccccccccccccccccccccccccccc |
+| cccccccc | ccccccccccccccccccccccccccccccccccc |
+| cccccccc | cccccccccccccccccBccccccccccccccccc |
+|          |                                     |
+ ------------------------------------------------
+"""
+
+

lib/utils/gametime.py

@@ -0,0 +1,179 @@
+"""
+The gametime module handles the global passage of time in the mud.
+
+It also supplies some useful methods to convert between
+in-mud time and real-world time as well allows to get the
+total runtime of the server and the current uptime.
+"""
+
+from time import time
+from django.conf import settings
+from src.scripts.scripts import Script
+from src.utils.create import create_script
+
+GAMETIME_SCRIPT_NAME = "sys_game_time"
+
+# Speed-up factor of the in-game time compared
+# to real time.
+
+TIMEFACTOR = settings.TIME_FACTOR
+
+# Common real-life time measure, in seconds.
+# You should not change this.
+
+REAL_MIN = 60.0  # seconds per minute in real world
+
+# Game-time units, in real-life seconds. These are supplied as
+# a convenient measure for determining the current in-game time,
+# e.g. when defining in-game events. The words month, week and year can
+# be used to mean whatever units of time are used in the game.
+
+MIN = settings.TIME_SEC_PER_MIN
+HOUR = MIN * settings.TIME_MIN_PER_HOUR
+DAY = HOUR * settings.TIME_HOUR_PER_DAY
+WEEK = DAY * settings.TIME_DAY_PER_WEEK
+MONTH = WEEK * settings.TIME_WEEK_PER_MONTH
+YEAR = MONTH * settings.TIME_MONTH_PER_YEAR
+
+# Cached time stamps
+SERVER_STARTTIME = time()
+SERVER_RUNTIME = 0.0
+
+
+class GameTime(Script):
+    """
+    This script repeatedly saves server times so
+    it can be retrieved after server downtime.
+    """
+    def at_script_creation(self):
+        """
+        Setup the script
+        """
+        self.key = GAMETIME_SCRIPT_NAME
+        self.desc = "Saves uptime/runtime"
+        self.interval = 60
+        self.persistent = True
+        self.start_delay = True
+        self.attributes.add("run_time", 0.0)  # OOC time
+        self.attributes.add("up_time", 0.0)  # OOC time
+
+    def at_repeat(self):
+        """
+        Called every minute to update the timers.
+        """
+        self.attributes.add("run_time", runtime())
+        self.attributes.add("up_time", uptime())
+
+    def at_start(self):
+        """
+        This is called once every server restart.
+        We reset the up time and load the relevant
+        times.
+        """
+        global SERVER_RUNTIME
+        SERVER_RUNTIME = self.attributes.get("run_time")
+
+def save():
+    "Force save of time. This is called by server when shutting down/reloading."
+    from src.scripts.models import ScriptDB
+    try:
+        script = ScriptDB.objects.get(db_key=GAMETIME_SCRIPT_NAME)
+        script.at_repeat()
+    except Exception:
+        from src.utils import logger
+        logger.log_trace()
+
+def _format(seconds, *divisors) :
+    """
+    Helper function. Creates a tuple of even dividends given
+    a range of divisors.
+
+    Inputs
+      seconds - number of seconds to format
+      *divisors - a number of integer dividends. The number of seconds will be
+                  integer-divided by the first number in this sequence, the remainder
+                  will be divided with the second and so on.
+    Output:
+        A tuple of length len(*args)+1, with the last element being the last remaining
+        seconds not evenly divided by the supplied dividends.
+
+    """
+    results = []
+    seconds = int(seconds)
+    for divisor in divisors:
+        results.append(seconds / divisor)
+        seconds %= divisor
+    results.append(seconds)
+    return tuple(results)
+
+
+# Access functions
+
+def runtime(format=False):
+    "Get the total runtime of the server since first start (minus downtimes)"
+    runtime = SERVER_RUNTIME + (time() - SERVER_STARTTIME)
+    if format:
+        return _format(runtime, 31536000, 2628000, 604800, 86400, 3600, 60)
+    return runtime
+
+def uptime(format=False):
+    "Get the current uptime of the server since last reload"
+    uptime = time() - SERVER_STARTTIME
+    if format:
+        return _format(uptime, 31536000, 2628000, 604800, 86400, 3600, 60)
+    return uptime
+
+def gametime(format=False):
+    "Get the total gametime of the server since first start (minus downtimes)"
+    gametime = runtime() * TIMEFACTOR
+    if format:
+        return _format(gametime, YEAR, MONTH, WEEK, DAY, HOUR, MIN)
+    return gametime
+
+
+def gametime_to_realtime(secs=0, mins=0, hrs=0, days=0,
+                         weeks=0, months=0, yrs=0, format=False):
+    """
+    This method helps to figure out the real-world time it will take until an
+    in-game time has passed. E.g. if an event should take place a month later
+    in-game, you will be able to find the number of real-world seconds this
+    corresponds to (hint: Interval events deal with real life seconds).
+
+    Example:
+     gametime_to_realtime(days=2) -> number of seconds in real life from
+                                now after which 2 in-game days will have passed.
+    """
+    realtime = (secs + mins * MIN + hrs * HOUR + days * DAY + weeks * WEEK + \
+                months * MONTH + yrs * YEAR) / TIMEFACTOR
+    if format:
+        return _format(realtime, 31536000, 2628000, 604800, 86400, 3600, 60)
+    return realtime
+
+
+def realtime_to_gametime(secs=0, mins=0, hrs=0, days=0,
+                         weeks=0, months=0, yrs=0, format=False):
+    """
+    This method calculates how much in-game time a real-world time
+    interval would correspond to. This is usually a lot less interesting
+    than the other way around.
+
+     Example:
+      realtime_to_gametime(days=2) -> number of game-world seconds
+                                      corresponding to 2 real days.
+    """
+    gametime = TIMEFACTOR * (secs + mins * 60 + hrs * 3600 + days * 86400 +
+                             weeks * 604800 + months * 2628000 + yrs * 31536000)
+    if format:
+        return _format(gametime, YEAR, MONTH, WEEK, DAY, HOUR, MIN)
+    return gametime
+
+
+# Time administration routines
+
+def init_gametime():
+    """
+    This is called once, when the server starts for the very first time.
+    """
+    # create the GameTime script and start it
+    game_time = create_script(GameTime)
+    game_time.start()

lib/utils/idmapper/EVENNIA.txt

@@ -0,0 +1,24 @@
+
+IDMAPPER
+--------
+
+https://github.com/dcramer/django-idmapper
+
+IDmapper (actually Django-idmapper) implements a custom Django model
+that is cached between database writes/read (SharedMemoryModel). It
+not only lowers memory consumption but most importantly allows for
+semi-persistance of properties on database model instances (something
+not guaranteed for normal Django models).
+
+Evennia makes a few modifications to the original IDmapper routines
+(we try to limit our modifications in order to make it easy to update
+it from upstream down the line). 
+
+- We change the caching from a WeakValueDictionary to a normal
+  dictionary. This is done because we use the models as semi-
+  persistent storage while the server was running. In some situations
+  the models would run out of scope and the WeakValueDictionary
+  then allowed them to be garbage collected. With this change they
+  are guaranteed to remain (which is good for persistence but 
+  potentially bad for memory consumption).
+- We add some caching/reset hooks called from the server side. 

lib/utils/idmapper/LICENSE

@@ -0,0 +1,9 @@
+Copyright (c) 2009, David Cramer <dcramer@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

lib/utils/idmapper/README.rst

@@ -0,0 +1,45 @@
+This fork of django-idmapper fixes some bugs that prevented the idmapper from
+being used in many instances. In particular, the caching manager is now inherited
+by SharedMemoryManager subclasses, and it is used when Django uses an automatic
+manager (see http://docs.djangoproject.com/en/dev/topics/db/managers/#controlling-automatic-manager-types). This means access through foreign keys now uses
+identity mapping.
+
+Tested with Django version 1.2 alpha 1 SVN-12375.
+
+My modifications are usually accompanied by comments marked with "CL:".
+
+Django Identity Mapper
+======================
+
+A pluggable Django application which allows you to explicitally mark your models to use an identity mapping pattern. This will share instances of the same model in memory throughout your interpreter.
+
+Please note, that deserialization (such as from the cache) will *not* use the identity mapper.
+
+Usage
+-----
+To use the shared memory model you simply need to inherit from it (instead of models.Model). This enable all queries (and relational queries) to this model to use the shared memory instance cache, effectively creating a single instance for each unique row (based on primary key) in the queryset.
+
+For example, if you want to simply mark all of your models as a SharedMemoryModel, you might as well just import it as models.
+::
+
+	from idmapper import models
+
+	class MyModel(models.SharedMemoryModel):
+	    name = models.CharField(...)
+
+Because the system is isolated, you may mix and match SharedMemoryModels with regular Models. The module idmapper.models imports everything from django.db.models and only adds SharedMemoryModel, so you can simply replace your import of models from django.db.
+::
+
+	from idmapper import models
+
+	class MyModel(models.SharedMemoryModel):
+	    name = models.CharField(...)
+	    fkey = models.ForeignKey('Other')
+
+	class Other(models.Model):
+	    name = models.CharField(...)
+
+References
+----------
+
+Original code and concept: http://code.djangoproject.com/ticket/17

lib/utils/idmapper/__init__.py

@@ -0,0 +1,41 @@
+import os.path
+import warnings
+
+__version__ = (0, 2)
+
+def _get_git_revision(path):
+    revision_file = os.path.join(path, 'refs', 'heads', 'master')
+    if not os.path.exists(revision_file):
+        return None
+    fh = open(revision_file, 'r')
+    try:
+        return fh.read()
+    finally:
+        fh.close()
+
+def get_revision():
+    """
+    :returns: Revision number of this branch/checkout, if available. None if
+        no revision number can be determined.
+    """
+    package_dir = os.path.dirname(__file__)
+    checkout_dir = os.path.normpath(os.path.join(package_dir, '..'))
+    path = os.path.join(checkout_dir, '.git')
+    if os.path.exists(path):
+        return _get_git_revision(path)
+    return None
+
+__build__ = get_revision()
+
+def lazy_object(location):
+    def inner(*args, **kwargs):
+        parts = location.rsplit('.', 1)
+        warnings.warn('`idmapper.%s` is deprecated. Please use `%s` instead.' % (parts[1], location), DeprecationWarning)
+        imp = __import__(parts[0], globals(), locals(), [parts[1]], -1)
+        func = getattr(imp, parts[1])
+        if callable(func):
+            return func(*args, **kwargs)
+        return func
+    return inner
+
+SharedMemoryModel = lazy_object('idmapper.models.SharedMemoryModel')

lib/utils/idmapper/base.py

@@ -0,0 +1,488 @@
+"""
+Django ID mapper
+
+Modified for Evennia by making sure that no model references
+leave caching unexpectedly (no use of WeakRefs).
+
+Also adds cache_size() for monitoring the size of the cache.
+"""
+
+import os, threading, gc, time
+#from twisted.internet import reactor
+#from twisted.internet.threads import blockingCallFromThread
+from weakref import WeakValueDictionary
+from twisted.internet.reactor import callFromThread
+from django.core.exceptions import ObjectDoesNotExist, FieldError
+from django.db.models.base import Model, ModelBase
+from django.db.models.signals import post_save, pre_delete, post_syncdb
+from src.utils import logger
+from src.utils.utils import dbref, get_evennia_pids, to_str
+
+from manager import SharedMemoryManager
+
+AUTO_FLUSH_MIN_INTERVAL = 60.0 * 5 # at least 5 mins between cache flushes
+
+
+_GA = object.__getattribute__
+_SA = object.__setattr__
+_DA = object.__delattr__
+
+# References to db-updated objects are stored here so the
+# main process can be informed to re-cache itself.
+PROC_MODIFIED_COUNT = 0
+PROC_MODIFIED_OBJS = WeakValueDictionary()
+
+# get info about the current process and thread; determine if our
+# current pid is different from the server PID (i.e.  # if we are in a
+# subprocess or not)
+_SELF_PID = os.getpid()
+_SERVER_PID, _PORTAL_PID = get_evennia_pids()
+_IS_SUBPROCESS = (_SERVER_PID and _PORTAL_PID) and not _SELF_PID in (_SERVER_PID, _PORTAL_PID)
+_IS_MAIN_THREAD = threading.currentThread().getName() == "MainThread"
+
+class SharedMemoryModelBase(ModelBase):
+    # CL: upstream had a __new__ method that skipped ModelBase's __new__ if
+    # SharedMemoryModelBase was not in the model class's ancestors. It's not
+    # clear what was the intended purpose, but skipping ModelBase.__new__
+    # broke things; in particular, default manager inheritance.
+
+    def __call__(cls, *args, **kwargs):
+        """
+        this method will either create an instance (by calling the default implementation)
+        or try to retrieve one from the class-wide cache by infering the pk value from
+        args and kwargs. If instance caching is enabled for this class, the cache is
+        populated whenever possible (ie when it is possible to infer the pk value).
+        """
+        def new_instance():
+            return super(SharedMemoryModelBase, cls).__call__(*args, **kwargs)
+
+        instance_key = cls._get_cache_key(args, kwargs)
+        # depending on the arguments, we might not be able to infer the PK, so in that case we create a new instance
+        if instance_key is None:
+            return new_instance()
+
+        cached_instance = cls.get_cached_instance(instance_key)
+        if cached_instance is None:
+            cached_instance = new_instance()
+            cls.cache_instance(cached_instance)
+        return cached_instance
+
+
+    def _prepare(cls):
+        """
+        Prepare the cache, making sure that proxies of the same db base
+        share the same cache.
+        """
+        def prep(dbmodel):
+            if not hasattr(dbmodel, "__instance_cache__"):
+                dbmodel.__instance_cache__ = {}
+                dbmodel._idmapper_recache_protection = False
+        if not cls._meta.proxy:
+            # non-proxy models get the full cache
+            prep(cls)
+        else:
+            # proxies get a reference to the cache
+            dbmodel = cls._meta.proxy_for_model
+            prep(dbmodel)
+            cls.__instance_cache__ = dbmodel.__instance_cache__
+            cls._idmapper_recache_protection = False
+        super(SharedMemoryModelBase, cls)._prepare()
+
+    def __new__(cls, name, bases, attrs):
+        """
+        Field shortcut creation:
+        Takes field names db_* and creates property wrappers named without the db_ prefix. So db_key -> key
+        This wrapper happens on the class level, so there is no overhead when creating objects. If a class
+        already has a wrapper of the given name, the automatic creation is skipped. Note: Remember to
+        document this auto-wrapping in the class header, this could seem very much like magic to the user otherwise.
+        """
+
+        attrs["typename"] = cls.__name__
+        attrs["path"] =  "%s.%s" % (attrs["__module__"], name)
+
+        # set up the typeclass handling only if a variable _is_typeclass is set on the class
+        def create_wrapper(cls, fieldname, wrappername, editable=True, foreignkey=False):
+            "Helper method to create property wrappers with unique names (must be in separate call)"
+            def _get(cls, fname):
+                "Wrapper for getting database field"
+                #print "_get:", fieldname, wrappername,_GA(cls,fieldname)
+                if _GA(cls, "_is_deleted"):
+                    raise ObjectDoesNotExist("Cannot access %s: Hosting object was already deleted." % fname)
+                return _GA(cls, fieldname)
+            def _get_foreign(cls, fname):
+                "Wrapper for returing foreignkey fields"
+                if _GA(cls, "_is_deleted"):
+                    raise ObjectDoesNotExist("Cannot access %s: Hosting object was already deleted." % fname)
+                value = _GA(cls, fieldname)
+                #print "_get_foreign:value:", value
+                try:
+                    return _GA(value, "typeclass")
+                except:
+                    return value
+            def _set_nonedit(cls, fname, value):
+                "Wrapper for blocking editing of field"
+                raise FieldError("Field %s cannot be edited." % fname)
+            def _set(cls, fname, value):
+                "Wrapper for setting database field"
+                if _GA(cls, "_is_deleted"):
+                    raise ObjectDoesNotExist("Cannot set %s to %s: Hosting object was already deleted!" % (fname, value))
+                _SA(cls, fname, value)
+                # only use explicit update_fields in save if we actually have a
+                # primary key assigned already (won't be set when first creating object)
+                update_fields = [fname] if _GA(cls, "_get_pk_val")(_GA(cls, "_meta")) is not None else None
+                _GA(cls, "save")(update_fields=update_fields)
+            def _set_foreign(cls, fname, value):
+                "Setter only used on foreign key relations, allows setting with #dbref"
+                if _GA(cls, "_is_deleted"):
+                    raise ObjectDoesNotExist("Cannot set %s to %s: Hosting object was already deleted!" % (fname, value))
+                try:
+                    value = _GA(value, "dbobj")
+                except AttributeError:
+                    pass
+                if isinstance(value, (basestring, int)):
+                    value = to_str(value, force_string=True)
+                    if (value.isdigit() or value.startswith("#")):
+                        # we also allow setting using dbrefs, if so we try to load the matching object.
+                        # (we assume the object is of the same type as the class holding the field, if
+                        # not a custom handler must be used for that field)
+                        dbid = dbref(value, reqhash=False)
+                        if dbid:
+                            model = _GA(cls, "_meta").get_field(fname).model
+                            try:
+                                value = model._default_manager.get(id=dbid)
+                            except ObjectDoesNotExist:
+                                # maybe it is just a name that happens to look like a dbid
+                                pass
+                _SA(cls, fname, value)
+                # only use explicit update_fields in save if we actually have a
+                # primary key assigned already (won't be set when first creating object)
+                update_fields = [fname] if _GA(cls, "_get_pk_val")(_GA(cls, "_meta")) is not None else None
+                _GA(cls, "save")(update_fields=update_fields)
+            def _del_nonedit(cls, fname):
+                "wrapper for not allowing deletion"
+                raise FieldError("Field %s cannot be edited." % fname)
+            def _del(cls, fname):
+                "Wrapper for clearing database field - sets it to None"
+                _SA(cls, fname, None)
+                update_fields = [fname] if _GA(cls, "_get_pk_val")(_GA(cls, "_meta")) is not None else None
+                _GA(cls, "save")(update_fields=update_fields)
+
+            # wrapper factories
+            fget = lambda cls: _get(cls, fieldname)
+            if not editable:
+                fset = lambda cls, val: _set_nonedit(cls, fieldname, val)
+            elif foreignkey:
+                fget = lambda cls: _get_foreign(cls, fieldname)
+                fset = lambda cls, val: _set_foreign(cls, fieldname, val)
+            else:
+                fset = lambda cls, val: _set(cls, fieldname, val)
+            fdel = lambda cls: _del(cls, fieldname) if editable else _del_nonedit(cls,fieldname)
+            # assigning
+            attrs[wrappername] = property(fget, fset, fdel)
+            #type(cls).__setattr__(cls, wrappername, property(fget, fset, fdel))#, doc))
+
+        # exclude some models that should not auto-create wrapper fields
+        if cls.__name__ in ("ServerConfig", "TypeNick"):
+            return
+        # dynamically create the wrapper properties for all fields not already handled (manytomanyfields are always handlers)
+        for fieldname, field in ((fname, field) for fname, field in attrs.items()
+                                  if fname.startswith("db_") and type(field).__name__ != "ManyToManyField"):
+            foreignkey = type(field).__name__ == "ForeignKey"
+            #print fieldname, type(field).__name__, field
+            wrappername = "dbid" if fieldname == "id" else fieldname.replace("db_", "", 1)
+            #print fieldname, wrappername
+            if wrappername not in attrs:
+                # makes sure not to overload manually created wrappers on the model
+                #print "wrapping %s -> %s" % (fieldname, wrappername)
+                create_wrapper(cls, fieldname, wrappername, editable=field.editable, foreignkey=foreignkey)
+
+        return super(SharedMemoryModelBase, cls).__new__(cls, name, bases, attrs)
+
+
+class SharedMemoryModel(Model):
+    # CL: setting abstract correctly to allow subclasses to inherit the default
+    # manager.
+    __metaclass__ = SharedMemoryModelBase
+
+    objects = SharedMemoryManager()
+
+    class Meta:
+        abstract = True
+
+    #def __init__(cls, *args, **kwargs):
+    #    super(SharedMemoryModel, cls).__init__(*args, **kwargs)
+    #    cls.__idmapper_recache_protection = False
+
+    @classmethod
+    def _get_cache_key(cls, args, kwargs):
+        """
+        This method is used by the caching subsystem to infer the PK value from the constructor arguments.
+        It is used to decide if an instance has to be built or is already in the cache.
+        """
+        result = None
+        # Quick hack for my composites work for now.
+        if hasattr(cls._meta, 'pks'):
+            pk = cls._meta.pks[0]
+        else:
+            pk = cls._meta.pk
+        # get the index of the pk in the class fields. this should be calculated *once*, but isn't atm
+        pk_position = cls._meta.fields.index(pk)
+        if len(args) > pk_position:
+            # if it's in the args, we can get it easily by index
+            result = args[pk_position]
+        elif pk.attname in kwargs:
+            # retrieve the pk value. Note that we use attname instead of name, to handle the case where the pk is a
+            # a ForeignKey.
+            result = kwargs[pk.attname]
+        elif pk.name != pk.attname and pk.name in kwargs:
+            # ok we couldn't find the value, but maybe it's a FK and we can find the corresponding object instead
+            result = kwargs[pk.name]
+
+        if result is not None and isinstance(result, Model):
+            # if the pk value happens to be a model instance (which can happen wich a FK), we'd rather use its own pk as the key
+            result = result._get_pk_val()
+        return result
+    #_get_cache_key = classmethod(_get_cache_key)
+
+    @classmethod
+    def get_cached_instance(cls, id):
+        """
+        Method to retrieve a cached instance by pk value. Returns None when not found
+        (which will always be the case when caching is disabled for this class). Please
+        note that the lookup will be done even when instance caching is disabled.
+        """
+        return cls.__instance_cache__.get(id)
+    #get_cached_instance = classmethod(get_cached_instance)
+
+    @classmethod
+    def cache_instance(cls, instance):
+        """
+        Method to store an instance in the cache.
+        """
+        if instance._get_pk_val() is not None:
+
+            cls.__instance_cache__[instance._get_pk_val()] = instance
+    #cache_instance = classmethod(cache_instance)
+
+    @classmethod
+    def get_all_cached_instances(cls):
+        "return the objects so far cached by idmapper for this class."
+        return cls.__instance_cache__.values()
+
+    @classmethod
+    def _flush_cached_by_key(cls, key, force=True):
+        "Remove the cached reference."
+        try:
+            if force or not cls._idmapper_recache_protection:
+                del cls.__instance_cache__[key]
+        except KeyError:
+            pass
+
+    @classmethod
+    def flush_cached_instance(cls, instance, force=True):
+        """
+        Method to flush an instance from the cache. The instance will
+        always be flushed from the cache, since this is most likely
+        called from delete(), and we want to make sure we don't cache
+        dead objects.
+
+        """
+        cls._flush_cached_by_key(instance._get_pk_val(), force=force)
+    #flush_cached_instance = classmethod(flush_cached_instance)
+
+    @classmethod
+    def flush_instance_cache(cls, force=False):
+        """
+        This will clean safe objects from the cache. Use force
+        keyword to remove all objects, safe or not.
+        """
+        if force:
+            cls.__instance_cache__ = {}
+        else:
+            cls.__instance_cache__ = dict((key, obj) for key, obj in cls.__instance_cache__.items()
+                                                      if obj._idmapper_recache_protection)
+    #flush_instance_cache = classmethod(flush_instance_cache)
+
+    # per-instance methods
+
+    def set_recache_protection(self, mode=True):
+        "set if this instance should be allowed to be recached."
+        self._idmapper_recache_protection = bool(mode)
+
+    def save(self, *args, **kwargs):
+        "save method tracking process/thread issues"
+
+        if _IS_SUBPROCESS:
+            # we keep a store of objects modified in subprocesses so
+            # we know to update their caches in the central process
+            global PROC_MODIFIED_COUNT, PROC_MODIFIED_OBJS
+            PROC_MODIFIED_COUNT += 1
+            PROC_MODIFIED_OBJS[PROC_MODIFIED_COUNT] = self
+
+        if _IS_MAIN_THREAD:
+            # in main thread - normal operation
+            super(SharedMemoryModel, self).save(*args, **kwargs)
+        else:
+            # in another thread; make sure to save in reactor thread
+            def _save_callback(cls, *args, **kwargs):
+                super(SharedMemoryModel, cls).save(*args, **kwargs)
+            callFromThread(_save_callback, self, *args, **kwargs)
+
+
+class WeakSharedMemoryModelBase(SharedMemoryModelBase):
+    """
+    Uses a WeakValue dictionary for caching instead of a regular one
+    """
+    def _prepare(cls):
+        super(WeakSharedMemoryModelBase, cls)._prepare()
+        cls.__instance_cache__ = WeakValueDictionary()
+        cls._idmapper_recache_protection = False
+
+
+class WeakSharedMemoryModel(SharedMemoryModel):
+    """
+    Uses a WeakValue dictionary for caching instead of a regular one
+    """
+    __metaclass__ = WeakSharedMemoryModelBase
+    class Meta:
+        abstract = True
+
+
+def flush_cache(**kwargs):
+    """
+    Flush idmapper cache. When doing so the cache will
+    look for a property _idmapper_cache_flush_safe on the
+    class/subclass instance and only flush if this
+    is True.
+
+    Uses a signal so we make sure to catch cascades.
+    """
+    def class_hierarchy(clslist):
+        """Recursively yield a class hierarchy"""
+        for cls in clslist:
+            subclass_list = cls.__subclasses__()
+            if subclass_list:
+                for subcls in class_hierarchy(subclass_list):
+                    yield subcls
+            else:
+                yield cls
+
+    for cls in class_hierarchy([SharedMemoryModel]):
+        cls.flush_instance_cache()
+    # run the python garbage collector
+    return gc.collect()
+#request_finished.connect(flush_cache)
+post_syncdb.connect(flush_cache)
+
+
+def flush_cached_instance(sender, instance, **kwargs):
+    """
+    Flush the idmapper cache only for a given instance
+    """
+    # XXX: Is this the best way to make sure we can flush?
+    if not hasattr(instance, 'flush_cached_instance'):
+        return
+    sender.flush_cached_instance(instance, force=True)
+pre_delete.connect(flush_cached_instance)
+
+
+def update_cached_instance(sender, instance, **kwargs):
+    """
+    Re-cache the given instance in the idmapper cache
+    """
+    if not hasattr(instance, 'cache_instance'):
+        return
+    sender.cache_instance(instance)
+post_save.connect(update_cached_instance)
+
+
+LAST_FLUSH = None
+def conditional_flush(max_rmem, force=False):
+    """
+    Flush the cache if the estimated memory usage exceeds max_rmem.
+
+    The flusher has a timeout to avoid flushing over and over
+    in particular situations (this means that for some setups
+    the memory usage will exceed the requirement and a server with
+    more memory is probably required for the given game)
+
+    force - forces a flush, regardless of timeout.
+    """
+    global LAST_FLUSH
+
+    def mem2cachesize(desired_rmem):
+        """
+        Estimate the size of the idmapper cache based on the memory
+        desired. This is used to optionally cap the cache size.
+
+        desired_rmem - memory in MB (minimum 50MB)
+
+        The formula is empirically estimated from usage tests (Linux)
+        and is
+            Ncache = RMEM - 35.0 / 0.0157
+        where RMEM is given in MB and Ncache is the size of the cache
+        for this memory usage. VMEM tends to be about 100MB higher
+        than RMEM for large memory usage.
+        """
+        vmem = max(desired_rmem, 50.0)
+        Ncache = int(abs(float(vmem) - 35.0) / 0.0157)
+        return Ncache
+
+    if not max_rmem:
+        # auto-flush is disabled
+        return
+
+    now = time.time()
+    if not LAST_FLUSH:
+        # server is just starting
+        LAST_FLUSH = now
+        return
+
+    if ((now - LAST_FLUSH) < AUTO_FLUSH_MIN_INTERVAL) and not force:
+        # too soon after last flush.
+        logger.log_warnmsg("Warning: Idmapper flush called more than "\
+                            "once in %s min interval. Check memory usage." % (AUTO_FLUSH_MIN_INTERVAL/60.0))
+        return
+
+    if os.name == "nt":
+        # we can't look for mem info in Windows at the moment
+        return
+
+    # check actual memory usage
+    Ncache_max = mem2cachesize(max_rmem)
+    Ncache, _ = cache_size()
+    actual_rmem = float(os.popen('ps -p %d -o %s | tail -1' % (os.getpid(), "rss")).read()) / 1000.0  # resident memory
+
+    if Ncache >= Ncache_max and actual_rmem > max_rmem * 0.9:
+        # flush cache when number of objects in cache is big enough and our
+        # actual memory use is within 10% of our set max
+        flush_cache()
+        LAST_FLUSH = now
+
+def cache_size(mb=True):
+    """
+    Calculate statistics about the cache.
+
+    Note: we cannot get reliable memory statistics from the cache -
+    whereas we could do getsizof each object in cache, the result is
+    highly imprecise and for a large number of object the result is
+    many times larger than the actual memory use of the entire server;
+    Python is clearly reusing memory behind the scenes that we cannot
+    catch in an easy way here.  Ideas are appreciated. /Griatch
+
+    Returns
+      total_num, {objclass:total_num, ...}
+    """
+    numtotal = [0] # use mutable to keep reference through recursion
+    classdict = {}
+    def get_recurse(submodels):
+        for submodel in submodels:
+            subclasses = submodel.__subclasses__()
+            if not subclasses:
+                num = len(submodel.get_all_cached_instances())
+                numtotal[0] += num
+                classdict[submodel.__name__] = num
+            else:
+                get_recurse(subclasses)
+    get_recurse(SharedMemoryModel.__subclasses__())
+    return numtotal[0], classdict

lib/utils/idmapper/manager.py

@@ -0,0 +1,44 @@
+from django.db.models.manager import Manager
+try:
+    from django.db import router
+except:
+    pass
+
+
+class SharedMemoryManager(Manager):
+    # CL: this ensures our manager is used when accessing instances via
+    # ForeignKey etc. (see docs)
+    use_for_related_fields = True
+
+    # CL: in the dev version of django, ReverseSingleRelatedObjectDescriptor
+    # will call us as:
+    #     rel_obj = rel_mgr.using(db).get(**params)
+    # We need to handle using, or the get method will be called on a vanilla
+    # queryset, and we won't get a change to use the cache.
+
+    #TODO - removing this for django1.7 - the call mentioned above doesn't happen
+    # anymore but is the cache still used? /Griatch
+    #def using(self, alias):
+    #    if alias == router.db_for_read(self.model):
+    #        # this should return a queryset!
+    #        return self
+    #    else:
+    #        return super(SharedMemoryManager, self).using(alias)
+
+    # TODO: improve on this implementation
+    # We need a way to handle reverse lookups so that this model can
+    # still use the singleton cache, but the active model isn't required
+    # to be a SharedMemoryModel.
+    def get(self, **kwargs):
+        items = kwargs.keys()
+        inst = None
+        if len(items) == 1:
+            # CL: support __exact
+            key = items[0]
+            if key.endswith('__exact'):
+                key = key[:-len('__exact')]
+            if key in ('pk', self.model._meta.pk.attname):
+                inst = self.model.get_cached_instance(kwargs[items[0]])
+        if inst is None:
+            inst = super(SharedMemoryManager, self).get(**kwargs)
+        return inst

lib/utils/idmapper/models.py

@@ -0,0 +1,2 @@
+from django.db.models import *
+from base import SharedMemoryModel, WeakSharedMemoryModel

lib/utils/idmapper/tests.py

@@ -0,0 +1,70 @@
+from django.test import TestCase
+
+from base import SharedMemoryModel
+from django.db import models
+
+class Category(SharedMemoryModel):
+    name = models.CharField(max_length=32)
+
+class RegularCategory(models.Model):
+    name = models.CharField(max_length=32)
+
+class Article(SharedMemoryModel):
+    name = models.CharField(max_length=32)
+    category = models.ForeignKey(Category)
+    category2 = models.ForeignKey(RegularCategory)
+
+class RegularArticle(models.Model):
+    name = models.CharField(max_length=32)
+    category = models.ForeignKey(Category)
+    category2 = models.ForeignKey(RegularCategory)
+
+class SharedMemorysTest(TestCase):
+    # TODO: test for cross model relation (singleton to regular)
+    
+    def setUp(self):
+        n = 0
+        category = Category.objects.create(name="Category %d" % (n,))
+        regcategory = RegularCategory.objects.create(name="Category %d" % (n,))
+        
+        for n in xrange(0, 10):
+            Article.objects.create(name="Article %d" % (n,), category=category, category2=regcategory)
+            RegularArticle.objects.create(name="Article %d" % (n,), category=category, category2=regcategory)
+    
+    def testSharedMemoryReferences(self):
+        article_list = Article.objects.all().select_related('category')
+        last_article = article_list[0]
+        for article in article_list[1:]:
+            self.assertEquals(article.category is last_article.category, True)
+            last_article = article
+
+    def testRegularReferences(self):
+        article_list = RegularArticle.objects.all().select_related('category')
+        last_article = article_list[0]
+        for article in article_list[1:]:
+            self.assertEquals(article.category2 is last_article.category2, False)
+            last_article = article
+
+    def testMixedReferences(self):
+        article_list = RegularArticle.objects.all().select_related('category')
+        last_article = article_list[0]
+        for article in article_list[1:]:
+            self.assertEquals(article.category is last_article.category, True)
+            last_article = article
+
+        article_list = Article.objects.all().select_related('category')
+        last_article = article_list[0]
+        for article in article_list[1:]:
+            self.assertEquals(article.category2 is last_article.category2, False)
+            last_article = article
+        
+    def testObjectDeletion(self):
+        # This must execute first so its guaranteed to be in memory.
+        article_list = list(Article.objects.all().select_related('category'))
+        
+        article = Article.objects.all()[0:1].get()
+        pk = article.pk
+        article.delete()
+        self.assertEquals(pk not in Article.__instance_cache__, True)
+        
+        

lib/utils/inlinefunc.py

@@ -0,0 +1,212 @@
+"""
+Inlinefunc
+
+This is a simple inline text language for use to custom-format text
+in Evennia. It is applied BEFORE ANSI/MUX parsing is applied.
+
+To activate Inlinefunc, settings.INLINEFUNC_ENABLED must be set.
+
+The format is straightforward:
+
+
+{funcname([arg1,arg2,...]) text {/funcname
+
+
+Example:
+    "This is {pad(50,c,-) a center-padded text{/pad of width 50."
+    ->
+    "This is -------------- a center-padded text--------------- of width 50."
+
+This can be inserted in any text, operated on by the parse_inlinefunc
+function.  funcname() (no space is allowed between the name and the
+argument tuple) is picked from a selection of valid functions from
+settings.INLINEFUNC_MODULES.
+
+Commands can be nested, and will applied inside-out. For correct
+parsing their end-tags must match the starting tags in reverse order.
+
+Example:
+    "The time is {pad(30){time(){/time{/padright now."
+    ->
+    "The time is         Oct 25, 11:09         right now."
+
+An inline function should have the following call signature:
+
+    def funcname(text, *args)
+
+where the text is always the part between {funcname(args) and
+{/funcname and the *args are taken from the appropriate part of the
+call. It is important that the inline function properly clean the
+incoming args, checking their type and replacing them with sane
+defaults if needed. If impossible to resolve, the unmodified text
+should be returned. The inlinefunc should never cause a traceback.
+
+"""
+
+import re
+from django.conf import settings
+from src.utils import utils
+
+
+# inline functions
+
+def pad(text, *args, **kwargs):
+    "Pad to width. pad(text, width=78, align='c', fillchar=' ')"
+    width = 78
+    align = 'c'
+    fillchar = ' '
+    for iarg, arg in enumerate(args):
+        if iarg == 0:
+            width = int(arg) if arg.isdigit() else width
+        elif iarg == 1:
+            align = arg if arg in ('c', 'l', 'r') else align
+        elif iarg == 2:
+            fillchar = arg[0]
+        else:
+            break
+    return utils.pad(text, width=width, align=align, fillchar=fillchar)
+
+def crop(text, *args, **kwargs):
+    "Crop to width. crop(text, width=78, suffix='[...]')"
+    width = 78
+    suffix = "[...]"
+    for iarg, arg in enumerate(args):
+        if iarg == 0:
+            width = int(arg) if arg.isdigit() else width
+        elif iarg == 1:
+            suffix = arg
+        else:
+            break
+    return utils.crop(text, width=width, suffix=suffix)
+
+def wrap(text, *args, **kwargs):
+    "Wrap/Fill text to width. fill(text, width=78, indent=0)"
+    width = 78
+    indent = 0
+    for iarg, arg in enumerate(args):
+        if iarg == 0:
+            width = int(arg) if arg.isdigit() else width
+        elif iarg == 1:
+            indent = int(arg) if arg.isdigit() else indent
+    return utils.wrap(text, width=width, indent=indent)
+
+def time(text, *args, **kwargs):
+    "Inserts current time"
+    import time
+    strformat = "%h %d, %H:%M"
+    if args and args[0]:
+        strformat = str(args[0])
+    return time.strftime(strformat)
+
+def you(text, *args, **kwargs):
+    "Inserts your name"
+    name = "You"
+    sess = kwargs.get("session")
+    if sess and sess.puppet:
+        name = sess.puppet.key
+    return name
+
+
+# load functions from module (including this one, if using default settings)
+_INLINE_FUNCS = {}
+for module in utils.make_iter(settings.INLINEFUNC_MODULES):
+    _INLINE_FUNCS.update(utils.all_from_module(module))
+_INLINE_FUNCS.pop("inline_func_parse", None)
+
+
+# dynamically build regexes for found functions
+_RE_FUNCFULL = r"\{%s\((.*?)\)(.*?){/%s"
+_RE_FUNCFULL_SINGLE = r"\{%s\((.*?)\)"
+_RE_FUNCSTART = r"\{((?:%s))"
+_RE_FUNCEND = r"\{/((?:%s))"
+_RE_FUNCSPLIT = r"(\{/*(?:%s)(?:\(.*?\))*)"
+_RE_FUNCCLEAN = r"\{%s\(.*?\)|\{/%s"
+
+_INLINE_FUNCS = dict((key, (func, re.compile(_RE_FUNCFULL % (key, key), re.DOTALL &  re.MULTILINE),
+                                  re.compile(_RE_FUNCFULL_SINGLE % key, re.DOTALL & re.MULTILINE)))
+                          for key, func in _INLINE_FUNCS.items() if callable(func))
+_FUNCSPLIT_REGEX = re.compile(_RE_FUNCSPLIT % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
+_FUNCSTART_REGEX = re.compile(_RE_FUNCSTART % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
+_FUNCEND_REGEX = re.compile(_RE_FUNCEND % r"|".join([key for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
+_FUNCCLEAN_REGEX = re.compile("|".join([_RE_FUNCCLEAN % (key, key) for key in _INLINE_FUNCS]), re.DOTALL & re.MULTILINE)
+
+
+# inline parser functions
+
+def _execute_inline_function(funcname, text, session):
+    """
+    Get the enclosed text between {funcname(...) and {/funcname
+    and execute the inline function to replace the whole block
+    with the result.
+    Note that this lookup is "dumb" - we just grab the first end
+    tag we find. So to work correctly this function must be called
+    "inside out" on a nested function tree, so each call only works
+    on a "flat" tag.
+    """
+    def subfunc(match):
+        "replace the entire block with the result of the function call"
+        args = [part.strip() for part in match.group(1).split(",")]
+        intext = match.group(2)
+        kwargs = {"session":session}
+        return _INLINE_FUNCS[funcname][0](intext, *args, **kwargs)
+    return _INLINE_FUNCS[funcname][1].sub(subfunc, text)
+
+def _execute_inline_single_function(funcname, text, session):
+    """
+    Get the arguments of a single function call (no matching end tag)
+    and execute it with an empty text input.
+    """
+    def subfunc(match):
+        "replace the single call with the result of the function call"
+        args = [part.strip() for part in match.group(1).split(",")]
+        kwargs = {"session":session}
+        return _INLINE_FUNCS[funcname][0]("", *args, **kwargs)
+    return _INLINE_FUNCS[funcname][2].sub(subfunc, text)
+
+def parse_inlinefunc(text, strip=False, session=None):
+    """
+    Parse inline function-replacement.
+
+    strip - remove all supported inlinefuncs from text
+    session - session calling for the parsing
+    """
+
+    if strip:
+        # strip all functions
+        return _FUNCCLEAN_REGEX.sub("", text)
+
+    stack = []
+    for part in _FUNCSPLIT_REGEX.split(text):
+        endtag = _FUNCEND_REGEX.match(part)
+        if endtag:
+            # an end tag
+            endname = endtag.group(1)
+            while stack:
+                new_part = stack.pop()
+                part = new_part + part  # add backwards -> fowards
+                starttag = _FUNCSTART_REGEX.match(new_part)
+                if starttag:
+                    startname = starttag.group(1)
+                    if startname == endname:
+                        part = _execute_inline_function(startname, part, session)
+                        break
+        stack.append(part)
+    # handle single functions without matching end tags; these are treated
+    # as being called with an empty string as text argument.
+    outstack = []
+    for part in _FUNCSPLIT_REGEX.split("".join(stack)):
+        starttag = _FUNCSTART_REGEX.match(part)
+        if starttag:
+            startname = starttag.group(1)
+            part = _execute_inline_single_function(startname, part, session)
+        outstack.append(part)
+
+    return "".join(outstack)
+
+def _test():
+    # this should all be handled
+    s = "This is a text with a{pad(78,c,-)text {pad(5)of{/pad {pad(30)nice{/pad size{/pad inside {pad(4,l)it{/pad."
+    s2 = "This is a text with a----------------text   of               nice              size---------------- inside it  ."
+    t = parse_inlinefunc(s)
+    assert(t == s2)
+    return t

lib/utils/logger.py

@@ -0,0 +1,144 @@
+"""
+Logging facilities
+
+These are thin wrappers on top of Twisted's
+logging facilities; logs are all directed
+either to stdout (if Evennia is running in
+interactive mode) or to game/logs.
+
+The log_file() function uses its own threading
+system to log to arbitrary files in game/logs.
+
+Note:
+All logging functions have two aliases,
+log_type() and log_typemsg(). This is for
+historical, back-compatible reasons.
+
+"""
+
+from datetime import datetime
+from traceback import format_exc
+from twisted.python import log
+from twisted.internet.threads import deferToThread
+
+
+def log_trace(errmsg=None):
+    """
+    Log a traceback to the log. This should be called
+    from within an exception. errmsg is optional and
+    adds an extra line with added info.
+    """
+    tracestring = format_exc()
+    try:
+        if tracestring:
+            for line in tracestring.splitlines():
+                log.msg('[::] %s' % line)
+        if errmsg:
+            try:
+                errmsg = str(errmsg)
+            except Exception, e:
+                errmsg = str(e)
+            for line in errmsg.splitlines():
+                log.msg('[EE] %s' % line)
+    except Exception:
+        log.msg('[EE] %s' % errmsg)
+log_tracemsg = log_trace
+
+
+def log_err(errmsg):
+    """
+    Prints/logs an error message to the server log.
+
+    errormsg: (string) The message to be logged.
+    """
+    try:
+        errmsg = str(errmsg)
+    except Exception, e:
+        errmsg = str(e)
+    for line in errmsg.splitlines():
+        log.msg('[EE] %s' % line)
+    #log.err('ERROR: %s' % (errormsg,))
+log_errmsg = log_err
+
+
+def log_warn(warnmsg):
+    """
+    Prints/logs any warnings that aren't critical but should be noted.
+
+    warnmsg: (string) The message to be logged.
+    """
+    try:
+        warnmsg = str(warnmsg)
+    except Exception, e:
+        warnmsg = str(e)
+    for line in warnmsg.splitlines():
+        log.msg('[WW] %s' % line)
+    #log.msg('WARNING: %s' % (warnmsg,))
+log_warnmsg = log_warn
+
+
+def log_info(infomsg):
+    """
+    Prints any generic debugging/informative info that should appear in the log.
+
+    infomsg: (string) The message to be logged.
+    """
+    try:
+        infomsg = str(infomsg)
+    except Exception, e:
+        infomsg = str(e)
+    for line in infomsg.splitlines():
+        log.msg('[..] %s' % line)
+log_infomsg = log_info
+
+
+def log_dep(depmsg):
+    """
+    Prints a deprecation message
+    """
+    try:
+        depmsg = str(depmsg)
+    except Exception, e:
+        depmsg = str(e)
+    for line in depmsg.splitlines():
+        log.msg('[DP] %s' % line)
+log_depmsg = log_dep
+
+
+# Arbitrary file logger
+
+LOG_FILE_HANDLES = {} # holds open log handles
+
+def log_file(msg, filename="game.log"):
+    """
+    Arbitrary file logger using threads.  Filename defaults to
+    'game.log'. All logs will appear in game/logs directory and log
+    entries will start on new lines following datetime info.
+    """
+    global LOG_FILE_HANDLES
+
+    def callback(filehandle, msg):
+        "Writing to file and flushing result"
+        msg = "\n%s [-] %s" % (datetime.now(), msg.strip())
+        filehandle.write(msg)
+        # since we don't close the handle, we need to flush
+        # manually or log file won't be written to until the
+        # write buffer is full.
+        filehandle.flush()
+    def errback(failure):
+        "Catching errors to normal log"
+        log_trace()
+
+    # save to game/logs/ directory
+    filename = "logs/" + filename
+
+    if filename in LOG_FILE_HANDLES:
+        filehandle = LOG_FILE_HANDLES[filename]
+    else:
+        try:
+            filehandle = open(filename, "a")
+            LOG_FILE_HANDLES[filename] = filehandle
+        except IOError:
+            log_trace()
+            return
+    deferToThread(callback, filehandle, msg).addErrback(errback)

lib/utils/picklefield.py

@@ -0,0 +1,279 @@
+#
+#  Copyright (c) 2009-2010 Gintautas Miliauskas
+#
+#   Permission is hereby granted, free of charge, to any person
+#   obtaining a copy of this software and associated documentation
+#   files (the "Software"), to deal in the Software without
+#   restriction, including without limitation the rights to use,
+#   copy, modify, merge, publish, distribute, sublicense, and/or sell
+#   copies of the Software, and to permit persons to whom the
+#   Software is furnished to do so, subject to the following
+#   conditions:
+#
+#   The above copyright notice and this permission notice shall be
+#   included in all copies or substantial portions of the Software.
+#
+#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#   OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#   OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Pickle field implementation for Django.
+
+Modified for Evennia by Griatch.
+
+"""
+from ast import literal_eval
+
+from copy import deepcopy
+from base64 import b64encode, b64decode
+from zlib import compress, decompress
+#import six # this is actually a pypy component, not in default syslib
+import django
+from django.core.exceptions import ValidationError
+from django.db import models
+
+# django 1.5 introduces force_text instead of force_unicode
+from django.forms import CharField, Textarea
+from django.forms.util import flatatt
+from django.utils.html import format_html
+
+from src.utils.dbserialize import from_pickle, to_pickle
+
+try:
+    from django.utils.encoding import force_text
+except ImportError:
+    from django.utils.encoding import force_unicode as force_text
+
+# python 3.x does not have cPickle module
+try:
+    from cPickle import loads, dumps # cpython 2.x
+except ImportError:
+    from pickle import loads, dumps # cpython 3.x, other interpreters
+
+DEFAULT_PROTOCOL = 2
+
+class PickledObject(str):
+    """
+    A subclass of string so it can be told whether a string is a pickled
+    object or not (if the object is an instance of this class then it must
+    [well, should] be a pickled one).
+
+    Only really useful for passing pre-encoded values to ``default``
+    with ``dbsafe_encode``, not that doing so is necessary. If you
+    remove PickledObject and its references, you won't be able to pass
+    in pre-encoded values anymore, but you can always just pass in the
+    python objects themselves.
+    """
+
+
+class _ObjectWrapper(object):
+    """
+    A class used to wrap object that have properties that may clash with the
+    ORM internals.
+
+    For example, objects with the `prepare_database_save` property such as
+    `django.db.Model` subclasses won't work under certain conditions and the
+    same apply for trying to retrieve any `callable` object.
+    """
+    __slots__ = ('_obj',)
+
+    def __init__(self, obj):
+        self._obj = obj
+
+
+def wrap_conflictual_object(obj):
+    if hasattr(obj, 'prepare_database_save') or callable(obj):
+        obj = _ObjectWrapper(obj)
+    return obj
+
+
+def dbsafe_encode(value, compress_object=False, pickle_protocol=DEFAULT_PROTOCOL):
+    # We use deepcopy() here to avoid a problem with cPickle, where dumps
+    # can generate different character streams for same lookup value if
+    # they are referenced differently.
+    # The reason this is important is because we do all of our lookups as
+    # simple string matches, thus the character streams must be the same
+    # for the lookups to work properly. See tests.py for more information.
+    value = dumps(deepcopy(value), protocol=pickle_protocol)
+    if compress_object:
+        value = compress(value)
+    value = b64encode(value).decode() # decode bytes to str
+    return PickledObject(value)
+
+
+def dbsafe_decode(value, compress_object=False):
+    value = value.encode() # encode str to bytes
+    value = b64decode(value)
+    if compress_object:
+        value = decompress(value)
+    return loads(value)
+
+
+def _get_subfield_superclass():
+    # hardcore trick to support django < 1.3 - there was something wrong with
+    # inheritance and SubfieldBase before django 1.3
+    # see https://github.com/django/django/commit/222c73261650201f5ce99e8dd4b1ce0d30a69eb4
+    if django.VERSION < (1,3):
+        return models.Field
+    # mimic six.with_metaclass
+    meta = models.SubfieldBase
+    base = models.Field
+    return meta("NewBase", (base,), {})
+    #return six.with_metaclass(models.SubfieldBase, models.Field)
+
+
+class PickledWidget(Textarea):
+    def render(self, name, value, attrs=None):
+        value = repr(value)
+        try:
+            literal_eval(value)
+        except ValueError:
+            return value
+
+        final_attrs = self.build_attrs(attrs, name=name)
+        return format_html('<textarea{0}>\r\n{1}</textarea>',
+                           flatatt(final_attrs),
+                           force_text(value))
+
+
+class PickledFormField(CharField):
+    widget = PickledWidget
+    default_error_messages = dict(CharField.default_error_messages)
+    default_error_messages['invalid'] = (
+        "This is not a Python Literal. You can store things like strings, "
+        "integers, or floats, but you must do it by typing them as you would "
+        "type them in the Python Interpreter. For instance, strings must be "
+        "surrounded by quote marks. We have converted it to a string for your "
+        "convenience. If it is acceptable, please hit save again.")
+
+    def __init__(self, *args, **kwargs):
+        # This needs to fall through to literal_eval.
+        kwargs['required'] = False
+        super(PickledFormField, self).__init__(*args, **kwargs)
+
+    def clean(self, value):
+        if value == '':
+            # Field was left blank. Make this None.
+            value = 'None'
+        try:
+            return literal_eval(value)
+        except (ValueError, SyntaxError):
+            raise ValidationError(self.error_messages['invalid'])
+
+
+class PickledObjectField(_get_subfield_superclass()):
+    """
+    A field that will accept *any* python object and store it in the
+    database. PickledObjectField will optionally compress its values if
+    declared with the keyword argument ``compress=True``.
+
+    Does not actually encode and compress ``None`` objects (although you
+    can still do lookups using None). This way, it is still possible to
+    use the ``isnull`` lookup type correctly.
+    """
+    __metaclass__ = models.SubfieldBase  # for django < 1.3
+
+    def __init__(self, *args, **kwargs):
+        self.compress = kwargs.pop('compress', False)
+        self.protocol = kwargs.pop('protocol', DEFAULT_PROTOCOL)
+        super(PickledObjectField, self).__init__(*args, **kwargs)
+
+    def get_default(self):
+        """
+        Returns the default value for this field.
+
+        The default implementation on models.Field calls force_unicode
+        on the default, which means you can't set arbitrary Python
+        objects as the default. To fix this, we just return the value
+        without calling force_unicode on it. Note that if you set a
+        callable as a default, the field will still call it. It will
+        *not* try to pickle and encode it.
+
+        """
+        if self.has_default():
+            if callable(self.default):
+                return self.default()
+            return self.default
+        # If the field doesn't have a default, then we punt to models.Field.
+        return super(PickledObjectField, self).get_default()
+
+    def to_python(self, value):
+        """
+        B64decode and unpickle the object, optionally decompressing it.
+
+        If an error is raised in de-pickling and we're sure the value is
+        a definite pickle, the error is allowed to propagate. If we
+        aren't sure if the value is a pickle or not, then we catch the
+        error and return the original value instead.
+
+        """
+        if value is not None:
+            try:
+                value = dbsafe_decode(value, self.compress)
+            except:
+                # If the value is a definite pickle; and an error is raised in
+                # de-pickling it should be allowed to propogate.
+                if isinstance(value, PickledObject):
+                    raise
+            else:
+                if isinstance(value, _ObjectWrapper):
+                    return value._obj
+        return value
+
+    def formfield(self, **kwargs):
+        return PickledFormField(**kwargs)
+
+    def pre_save(self, model_instance, add):
+        value = super(PickledObjectField, self).pre_save(model_instance, add)
+        return wrap_conflictual_object(value)
+
+    def get_db_prep_value(self, value, connection=None, prepared=False):
+        """
+        Pickle and b64encode the object, optionally compressing it.
+
+        The pickling protocol is specified explicitly (by default 2),
+        rather than as -1 or HIGHEST_PROTOCOL, because we don't want the
+        protocol to change over time. If it did, ``exact`` and ``in``
+        lookups would likely fail, since pickle would now be generating
+        a different string.
+
+        """
+        if value is not None and not isinstance(value, PickledObject):
+            # We call force_text here explicitly, so that the encoded string
+            # isn't rejected by the postgresql_psycopg2 backend. Alternatively,
+            # we could have just registered PickledObject with the psycopg
+            # marshaller (telling it to store it like it would a string), but
+            # since both of these methods result in the same value being stored,
+            # doing things this way is much easier.
+            value = force_text(dbsafe_encode(value, self.compress, self.protocol))
+        return value
+
+    def value_to_string(self, obj):
+        value = self._get_val_from_obj(obj)
+        return self.get_db_prep_value(value)
+
+    def get_internal_type(self):
+        return 'TextField'
+
+    def get_db_prep_lookup(self, lookup_type, value, connection=None, prepared=False):
+        if lookup_type not in ['exact', 'in', 'isnull']:
+            raise TypeError('Lookup type %s is not supported.' % lookup_type)
+        # The Field model already calls get_db_prep_value before doing the
+        # actual lookup, so all we need to do is limit the lookup types.
+        return super(PickledObjectField, self).get_db_prep_lookup(
+            lookup_type, value, connection=connection, prepared=prepared)
+
+
+# South support; see http://south.aeracode.org/docs/tutorial/part4.html#simple-inheritance
+try:
+    from south.modelsinspector import add_introspection_rules
+except ImportError:
+    pass
+else:
+    add_introspection_rules([], [r"^src\.utils\.picklefield\.PickledObjectField"])

lib/utils/search.py

@@ -0,0 +1,227 @@
+
+"""
+This is a convenient container gathering all the main
+search methods for the various database tables.
+
+It is intended to be used e.g. as
+
+> from src.utils import search
+> match = search.objects(...)
+
+Note that this is not intended to be a complete listing of all search
+methods! You need to refer to the respective manager to get all
+possible search methods. To get to the managers from your code, import
+the database model and call its 'objects' property.
+
+Also remember that all commands in this file return lists (also if
+there is only one match) unless noted otherwise.
+
+Example: To reach the search method 'get_object_with_player'
+         in src/objects/managers.py:
+
+> from src.objects.models import ObjectDB
+> match = Object.objects.get_object_with_player(...)
+
+
+"""
+
+# Import the manager methods to be wrapped
+
+from django.contrib.contenttypes.models import ContentType
+
+# limit symbol import from API
+__all__ = ("search_object", "search_player", "search_script",
+           "search_message", "search_channel", "search_help_entry",
+           "search_object_tag", "search_script_tag", "search_player_tag",
+           "search_channel_tag")
+
+
+# import objects this way to avoid circular import problems
+ObjectDB = ContentType.objects.get(app_label="objects", model="objectdb").model_class()
+PlayerDB = ContentType.objects.get(app_label="players", model="playerdb").model_class()
+ScriptDB = ContentType.objects.get(app_label="scripts", model="scriptdb").model_class()
+Msg = ContentType.objects.get(app_label="comms", model="msg").model_class()
+Channel = ContentType.objects.get(app_label="comms", model="channeldb").model_class()
+HelpEntry = ContentType.objects.get(app_label="help", model="helpentry").model_class()
+Tag = ContentType.objects.get(app_label="typeclasses", model="tag").model_class()
+
+#
+# Search objects as a character
+#
+# NOTE: A more powerful wrapper of this method
+#  is reachable from within each command class
+#  by using self.caller.search()!
+#
+#    def object_search(self, ostring=None,
+#                      attribute_name=None,
+#                      typeclass=None,
+#                      candidates=None,
+#                      exact=True):
+#
+#        Search globally or in a list of candidates and return results.
+#        The result is always a list of Objects (or the empty list)
+#
+#        Arguments:
+#        ostring: (str) The string to compare names against. By default (if
+#                  not attribute_name is set), this will search object.key
+#                  and object.aliases in order. Can also be on the form #dbref,
+#                  which will, if exact=True be matched against primary key.
+#        attribute_name: (str): Use this named ObjectAttribute to match ostring
+#                        against, instead of the defaults.
+#        typeclass (str or TypeClass): restrict matches to objects having
+#                  this typeclass. This will help speed up global searches.
+#        candidates (list obj ObjectDBs): If supplied, search will only be
+#                  performed among the candidates in this list. A common list
+#                  of candidates is the contents of the current location.
+#        exact (bool): Match names/aliases exactly or partially. Partial
+#                  matching matches the beginning of words in the names/aliases,
+#                  using a matching routine to separate multiple matches in
+#                  names with multiple components (so "bi sw" will match
+#                  "Big sword"). Since this is more expensive than exact
+#                  matching, it is recommended to be used together with
+#                  the objlist keyword to limit the number of possibilities.
+#                  This keyword has no meaning if attribute_name is set.
+#
+#        Returns:
+#        A list of matching objects (or a list with one unique match)
+#    def object_search(self, ostring, caller=None,
+#                      candidates=None,
+#                      attribute_name=None):
+#
+search_object = ObjectDB.objects.object_search
+search_objects = search_object
+object_search = search_object
+objects = search_objects
+
+#
+# Search for players
+#
+# def player_search(self, ostring):
+#     """
+#     Searches for a particular player by name or
+#     database id.
+#
+#     ostring = a string or database id.
+#     """
+
+search_player = PlayerDB.objects.player_search
+search_players = search_player
+player_search = search_player
+players = search_players
+
+#
+#   Searching for scripts
+#
+# def script_search(self, ostring, obj=None, only_timed=False):
+#     """
+#     Search for a particular script.
+#
+#     ostring - search criterion - a script ID or key
+#     obj - limit search to scripts defined on this object
+#     only_timed - limit search only to scripts that run
+#                  on a timer.
+#     """
+
+search_script = ScriptDB.objects.script_search
+search_scripts = search_script
+script_search = search_script
+scripts = search_scripts
+#
+# Searching for communication messages
+#
+#
+# def message_search(self, sender=None, receiver=None, channel=None, freetext=None):
+#     """
+#     Search the message database for particular messages. At least one
+#     of the arguments must be given to do a search.
+#
+#     sender - get messages sent by a particular player
+#     receiver - get messages received by a certain player
+#     channel - get messages sent to a particular channel
+#     freetext - Search for a text string in a message.
+#                NOTE: This can potentially be slow, so make sure to supply
+#                one of the other arguments to limit the search.
+#     """
+
+search_message = Msg.objects.message_search
+search_messages = search_message
+message_search = search_message
+messages = search_messages
+
+#
+# Search for Communication Channels
+#
+# def channel_search(self, ostring)
+#     """
+#     Search the channel database for a particular channel.
+#
+#     ostring - the key or database id of the channel.
+#     """
+
+search_channel = Channel.objects.channel_search
+search_channels = search_channel
+channel_search = search_channel
+channels = search_channels
+
+#
+# Find help entry objects.
+#
+# def search_help(self, ostring, help_category=None):
+#     """
+#     Retrieve a search entry object.
+#
+#     ostring - the help topic to look for
+#     category - limit the search to a particular help topic
+#     """
+
+search_help_entry = HelpEntry.objects.search_help
+search_help_entries = search_help_entry
+help_entry_search = search_help_entry
+help_entries = search_help_entries
+
+
+# Locate Attributes
+
+#    search_object_attribute(key, category, value, strvalue) (also search_attribute works)
+#    search_player_attribute(key, category, value, strvalue) (also search_attribute works)
+#    search_script_attribute(key, category, value, strvalue) (also search_attribute works)
+#    search_channel_attribute(key, category, value, strvalue) (also search_attribute works)
+
+# Note that these return the object attached to the Attribute,
+# not the attribute object itself (this is usually what you want)
+
+def search_object_attribute(key=None, category=None, value=None, strvalue=None):
+    return ObjectDB.objects.get_by_attribute(key=key, category=category, value=value, strvalue=strvalue)
+def search_player_attribute(key=None, category=None, value=None, strvalue=None):
+    return PlayerDB.objects.get_by_attribute(key=key, category=category, value=value, strvalue=strvalue)
+def search_script_attribute(key=None, category=None, value=None, strvalue=None):
+    return ScriptDB.objects.get_by_attribute(key=key, category=category, value=value, strvalue=strvalue)
+def search_channel_attribute(key=None, category=None, value=None, strvalue=None):
+    return Channel.objects.get_by_attribute(key=key, category=category, value=value, strvalue=strvalue)
+
+# search for attribute objects
+search_attribute_object = ObjectDB.objects.get_attribute
+
+# Locate Tags
+
+#    search_object_tag(key=None, category=None) (also search_tag works)
+#    search_player_tag(key=None, category=None)
+#    search_script_tag(key=None, category=None)
+#    search_channel_tag(key=None, category=None)
+
+# Note that this returns the object attached to the tag, not the tag
+# object itself (this is usually what you want)
+def search_object_tag(key=None, category=None):
+    return ObjectDB.objects.get_by_tag(key=key, category=category)
+search_tag = search_object_tag # this is the most common case
+def search_player_tag(key=None, category=None):
+    return PlayerDB.objects.get_by_tag(key=key, category=category)
+def search_script_tag(key=None, category=None):
+    return ScriptDB.objects.get_by_tag(key=key, category=category)
+def search_channel_tag(key=None, category=None):
+    return Channel.objects.get_by_tag(key=key, category=category)
+
+# search for tag objects
+search_tag_object = ObjectDB.objects.get_tag
+
+

lib/utils/spawner.py

@@ -0,0 +1,258 @@
+"""
+Spawner
+
+The spawner takes input files containing object definitions in
+dictionary forms. These use a prototype architechture to define
+unique objects without having to make a Typeclass for each.
+
+The main function is spawn(*prototype), where the prototype
+is a dictionary like this:
+
+GOBLIN = {
+ "typeclass": "game.gamesrc.objects.objects.Monster",
+ "key": "goblin grunt",
+ "health": lambda: randint(20,30),
+ "resists": ["cold", "poison"],
+ "attacks": ["fists"],
+ "weaknesses": ["fire", "light"]
+ }
+
+Possible keywords are:
+    prototype - string parent prototype
+    key - string, the main object identifier
+    typeclass - string, if not set, will use settings.BASE_OBJECT_TYPECLASS
+    location - this should be a valid object or #dbref
+    home - valid object or #dbref
+    destination - only valid for exits (object or dbref)
+
+    permissions - string or list of permission strings
+    locks - a lock-string
+    aliases - string or list of strings
+
+    ndb_<name> - value of a nattribute (ndb_ is stripped)
+    any other keywords are interpreted as Attributes and their values.
+
+Each value can also be a callable that takes no arguments. It should
+return the value to enter into the field and will be called every time
+the prototype is used to spawn an object.
+
+By specifying a prototype, the child will inherit all prototype slots
+it does not explicitly define itself, while overloading those that it
+does specify.
+
+GOBLIN_WIZARD = {
+ "prototype": GOBLIN,
+ "key": "goblin wizard",
+ "spells": ["fire ball", "lighting bolt"]
+ }
+
+GOBLIN_ARCHER = {
+ "prototype": GOBLIN,
+ "key": "goblin archer",
+ "attacks": ["short bow"]
+}
+
+One can also have multiple prototypes. These are inherited from the
+left, with the ones further to the right taking precedence.
+
+ARCHWIZARD = {
+ "attack": ["archwizard staff", "eye of doom"]
+
+GOBLIN_ARCHWIZARD = {
+ "key" : "goblin archwizard"
+ "prototype": (GOBLIN_WIZARD, ARCHWIZARD),
+}
+
+The goblin archwizard will have some different attacks, but will
+otherwise have the same spells as a goblin wizard who in turn shares
+many traits with a normal goblin.
+
+"""
+
+import os, sys, copy
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'game.settings'
+
+from django.conf import settings
+from random import randint
+from src.objects.models import ObjectDB
+from src.utils.utils import make_iter, all_from_module, dbid_to_obj
+
+_CREATE_OBJECT_KWARGS = ("key", "location", "home", "destination")
+
+_handle_dbref = lambda inp: dbid_to_obj(inp, ObjectDB)
+
+
+def _validate_prototype(key, prototype, protparents, visited):
+    "Run validation on a prototype, checking for inifinite regress"
+    assert isinstance(prototype, dict)
+    if id(prototype) in visited:
+        raise RuntimeError("%s has infinite nesting of prototypes." % key or prototype)
+    visited.append(id(prototype))
+    protstrings = prototype.get("prototype")
+    if protstrings:
+        for protstring in make_iter(protstrings):
+            if key is not None and protstring == key:
+                raise RuntimeError("%s tries to prototype itself." % key or prototype)
+            protparent = protparents.get(protstring)
+            if not protparent:
+                raise RuntimeError("%s's prototype '%s' was not found." % (key or prototype, protstring))
+            _validate_prototype(protstring, protparent, protparents, visited)
+
+
+def _get_prototype(dic, prot, protparents):
+    """
+    Recursively traverse a prototype dictionary,
+    including multiple inheritance. Use _validate_prototype
+    before this, we don't check for infinite recursion here.
+    """
+    if "prototype" in dic:
+        # move backwards through the inheritance
+        for prototype in make_iter(dic["prototype"]):
+            # Build the prot dictionary in reverse order, overloading
+            new_prot = _get_prototype(protparents.get(prototype, {}), prot, protparents)
+            prot.update(new_prot)
+    prot.update(dic)
+    prot.pop("prototype", None) # we don't need this anymore
+    return prot
+
+def _batch_create_object(*objparams):
+    """
+    This is a cut-down version of the create_object() function,
+    optimized for speed. It does NOT check and convert various input
+    so make sure the spawned Typeclass works before using this!
+
+    Input:
+    objsparams - each argument should be a tuple of arguments for the respective
+                 creation/add handlers in the following order:
+                    (create, permissions, locks, aliases, nattributes, attributes)
+    Returns:
+    A list of created objects
+    """
+
+    # bulk create all objects in one go
+    dbobjs = [ObjectDB(**objparam[0]) for objparam in objparams]
+    # unfortunately this doesn't work since bulk_create don't creates pks;
+    # the result are double objects at the next stage
+    #dbobjs = _ObjectDB.objects.bulk_create(dbobjs)
+
+    objs = []
+    for iobj, obj in enumerate(dbobjs):
+        # call all setup hooks on each object
+        objparam = objparams[iobj]
+        # setup
+        obj._createdict = {"pernmissions": objparam[1],
+                           "locks": objparam[2],
+                           "aliases": objparam[3],
+                           "attributes": objparam[4],
+                           "nattributes": objparam[5]}
+        # this triggers all hooks
+        obj.save()
+    return objs
+
+
+def spawn(*prototypes, **kwargs):
+    """
+    Spawn a number of prototyped objects. Each argument should be a
+    prototype dictionary.
+
+    keyword args:
+        prototype_modules - a python-path to a
+            prototype module, or a list of such paths. These will be used
+            to build the global protparents dictionary accessible by the
+            input prototypes. If not given, it will instead look for modules
+            defined by settings.PROTOTYPE_MODULES.
+        prototype_parents - a dictionary holding a custom prototype-parent dictionary. Will
+                      overload same-named prototypes from prototype_modules.
+        return_prototypes - only return a list of the prototype-parents
+                            (no object creation happens)
+    """
+
+    protparents = {}
+    protmodules = make_iter(kwargs.get("prototype_modules", []))
+    if not protmodules and hasattr(settings, "PROTOTYPE_MODULES"):
+        protmodules = make_iter(settings.PROTOTYPE_MODULES)
+    for prototype_module in protmodules:
+        protparents.update(dict((key, val)
+                for key, val in all_from_module(prototype_module).items() if isinstance(val, dict)))
+    #overload module's protparents with specifically given protparents
+    protparents.update(kwargs.get("prototype_parents", {}))
+    for key, prototype in protparents.items():
+        _validate_prototype(key, prototype, protparents, [])
+
+    if "return_prototypes" in kwargs:
+        # only return the parents
+        return copy.deepcopy(protparents)
+
+    objsparams = []
+    for prototype in prototypes:
+
+        _validate_prototype(None, prototype, protparents, [])
+        prot = _get_prototype(prototype, {}, protparents)
+        if not prot:
+            continue
+
+        # extract the keyword args we need to create the object itself
+        create_kwargs = {}
+        create_kwargs["db_key"] = prot.pop("key", "Spawned Object %06i" % randint(1,100000))
+        create_kwargs["db_location"] = _handle_dbref(prot.pop("location", None))
+        create_kwargs["db_home"] = _handle_dbref(prot.pop("home", settings.DEFAULT_HOME))
+        create_kwargs["db_destination"] = _handle_dbref(prot.pop("destination", None))
+        create_kwargs["db_typeclass_path"] = prot.pop("typeclass", settings.BASE_OBJECT_TYPECLASS)
+
+        # extract calls to handlers
+        permission_string = prot.pop("permissions", "")
+        lock_string = prot.pop("locks", "")
+        alias_string = prot.pop("aliases", "")
+
+        # extract ndb assignments
+        nattributes = dict((key.split("_", 1)[1], value if callable(value) else value)
+                            for key, value in prot.items() if key.startswith("ndb_"))
+
+        # the rest are attributes
+        attributes = dict((key, value() if callable(value) else value)
+                           for key, value in prot.items()
+                           if not (key in _CREATE_OBJECT_KWARGS or key in nattributes))
+
+        # pack for call into _batch_create_object
+        objsparams.append( (create_kwargs, permission_string, lock_string,
+                            alias_string, nattributes, attributes) )
+
+    return _batch_create_object(*objsparams)
+
+
+if __name__ == "__main__":
+    # testing
+
+    protparents = {
+            "NOBODY": {},
+            #"INFINITE" : {
+            #    "prototype":"INFINITE"
+            #},
+            "GOBLIN" : {
+             "key": "goblin grunt",
+             "health": lambda: randint(20,30),
+             "resists": ["cold", "poison"],
+             "attacks": ["fists"],
+             "weaknesses": ["fire", "light"]
+             },
+            "GOBLIN_WIZARD" : {
+             "prototype": "GOBLIN",
+             "key": "goblin wizard",
+             "spells": ["fire ball", "lighting bolt"]
+             },
+            "GOBLIN_ARCHER" : {
+             "prototype": "GOBLIN",
+             "key": "goblin archer",
+             "attacks": ["short bow"]
+            },
+            "ARCHWIZARD" : {
+             "attacks": ["archwizard staff"],
+            },
+            "GOBLIN_ARCHWIZARD" : {
+             "key": "goblin archwizard",
+             "prototype" : ("GOBLIN_WIZARD", "ARCHWIZARD")
+            }
+        }
+    # test
+    print [o.key for o in spawn(protparents["GOBLIN"], protparents["GOBLIN_ARCHWIZARD"], prototype_parents=protparents)]

lib/utils/tests.py

@@ -0,0 +1,121 @@
+import re
+
+try:
+    from django.utils.unittest import TestCase
+except ImportError:
+    from django.test import TestCase
+
+from ansi import ANSIString
+
+
+class ANSIStringTestCase(TestCase):
+    def checker(self, ansi, raw, clean):
+        """
+        Verifies the raw and clean strings of an ANSIString match expected
+        output.
+        """
+        self.assertEqual(unicode(ansi.clean()), clean)
+        self.assertEqual(unicode(ansi.raw()), raw)
+
+    def table_check(self, ansi, char, code):
+        """
+        Verifies the indexes in an ANSIString match what they should.
+        """
+        self.assertEqual(ansi._char_indexes, char)
+        self.assertEqual(ansi._code_indexes, code)
+
+    def test_instance(self):
+        """
+        Make sure the ANSIString is always constructed correctly.
+        """
+        clean = u'This isA{r testTest'
+        encoded = u'\x1b[1m\x1b[32mThis is\x1b[1m\x1b[31mA{r test\x1b[0mTest\x1b[0m'
+        target = ANSIString(r'{gThis is{rA{{r test{nTest{n')
+        char_table = [9, 10, 11, 12, 13, 14, 15, 25, 26, 27, 28, 29, 30, 31,
+                      32, 37, 38, 39, 40]
+        code_table = [0, 1, 2, 3, 4, 5, 6, 7, 8, 16, 17, 18, 19, 20, 21, 22,
+                      23, 24, 33, 34, 35, 36, 41, 42, 43, 44]
+        self.checker(target, encoded, clean)
+        self.table_check(target, char_table, code_table)
+        self.checker(ANSIString(target), encoded, clean)
+        self.table_check(ANSIString(target), char_table, code_table)
+        self.checker(ANSIString(encoded, decoded=True), encoded, clean)
+        self.table_check(ANSIString(encoded, decoded=True), char_table,
+                         code_table)
+        self.checker(ANSIString('Test'), u'Test', u'Test')
+        self.table_check(ANSIString('Test'), [0, 1, 2, 3], [])
+        self.checker(ANSIString(''), u'', u'')
+
+    def test_slice(self):
+        """
+        Verifies that slicing an ANSIString results in expected color code
+        distribution.
+        """
+        target = ANSIString(r'{gTest{rTest{n')
+        result = target[:3]
+        self.checker(result, u'\x1b[1m\x1b[32mTes', u'Tes')
+        result = target[:4]
+        self.checker(result, u'\x1b[1m\x1b[32mTest\x1b[1m\x1b[31m', u'Test')
+        result = target[:]
+        self.checker(
+            result,
+            u'\x1b[1m\x1b[32mTest\x1b[1m\x1b[31mTest\x1b[0m',
+            u'TestTest')
+        result = target[:-1]
+        self.checker(
+            result,
+            u'\x1b[1m\x1b[32mTest\x1b[1m\x1b[31mTes',
+            u'TestTes')
+        result = target[0:0]
+        self.checker(
+            result,
+            u'',
+            u'')
+
+    def test_split(self):
+        """
+        Verifies that re.split and .split behave similarly and that color
+        codes end up where they should.
+        """
+        target = ANSIString("{gThis is {nA split string{g")
+        first = (u'\x1b[1m\x1b[32mThis is \x1b[0m', u'This is ')
+        second = (u'\x1b[1m\x1b[32m\x1b[0m split string\x1b[1m\x1b[32m',
+                  u' split string')
+        re_split = re.split('A', target)
+        normal_split = target.split('A')
+        self.assertEqual(re_split, normal_split)
+        self.assertEqual(len(normal_split), 2)
+        self.checker(normal_split[0], *first)
+        self.checker(normal_split[1], *second)
+
+    def test_join(self):
+        """
+        Verify that joining a set of ANSIStrings works.
+        """
+        # This isn't the desired behavior, but the expected one. Python
+        # concatinates the in-memory representation with the built-in string's
+        # join.
+        l = [ANSIString("{gTest{r") for s in range(0, 3)]
+        # Force the generator to be evaluated.
+        result = "".join(l)
+        self.assertEqual(unicode(result), u'TestTestTest')
+        result = ANSIString("").join(l)
+        self.checker(result, u'\x1b[1m\x1b[32mTest\x1b[1m\x1b[31m\x1b[1m\x1b'
+                             u'[32mTest\x1b[1m\x1b[31m\x1b[1m\x1b[32mTest'
+                             u'\x1b[1m\x1b[31m', u'TestTestTest')
+
+    def test_len(self):
+        """
+        Make sure that length reporting on ANSIStrings does not include
+        ANSI codes.
+        """
+        self.assertEqual(len(ANSIString('{gTest{n')), 4)
+
+    def test_capitalize(self):
+        """
+        Make sure that capitalization works. This is the simplest of the
+        _transform functions.
+        """
+        target = ANSIString('{gtest{n')
+        result = u'\x1b[1m\x1b[32mTest\x1b[0m'
+        self.checker(target.capitalize(), result, u'Test')

lib/utils/text2html.py

@@ -0,0 +1,177 @@
+
+"""
+ANSI -> html converter
+
+Credit for original idea and implementation
+goes to Muhammad Alkarouri and his
+snippet #577349 on http://code.activestate.com.
+
+(extensively modified by Griatch 2010)
+"""
+
+import re
+import cgi
+from ansi import *
+
+
+class TextToHTMLparser(object):
+    """
+    This class describes a parser for converting from ansi to html.
+    """
+
+    tabstop = 4
+    # mapping html color name <-> ansi code.
+    hilite = ANSI_HILITE
+    normal = ANSI_NORMAL
+    underline = ANSI_UNDERLINE
+    colorcodes = [
+            ('red', hilite + ANSI_RED),
+            ('maroon', ANSI_RED),
+            ('lime', hilite + ANSI_GREEN),
+            ('green', ANSI_GREEN),
+            ('yellow', hilite + ANSI_YELLOW),
+            ('olive', ANSI_YELLOW),
+            ('blue', hilite + ANSI_BLUE),
+            ('navy', ANSI_BLUE),
+            ('magenta', hilite + ANSI_MAGENTA),
+            ('purple', ANSI_MAGENTA),
+            ('cyan', hilite + ANSI_CYAN),
+            ('teal', ANSI_CYAN),
+            ('white', hilite + ANSI_WHITE),  # pure white
+            ('gray', ANSI_WHITE),  # light grey
+            ('dimgray', hilite + ANSI_BLACK),  # dark grey
+            ('black', ANSI_BLACK),  # pure black
+    ]
+    colorback = [
+            ('bgred', hilite + ANSI_BACK_RED),
+            ('bgmaroon', ANSI_BACK_RED),
+            ('bglime', hilite + ANSI_BACK_GREEN),
+            ('bggreen', ANSI_BACK_GREEN),
+            ('bgyellow', hilite + ANSI_BACK_YELLOW),
+            ('bgolive', ANSI_BACK_YELLOW),
+            ('bgblue', hilite + ANSI_BACK_BLUE),
+            ('bgnavy', ANSI_BACK_BLUE),
+            ('bgmagenta', hilite + ANSI_BACK_MAGENTA),
+            ('bgpurple', ANSI_BACK_MAGENTA),
+            ('bgcyan', hilite + ANSI_BACK_CYAN),
+            ('bgteal', ANSI_BACK_CYAN),
+            ('bgwhite', hilite + ANSI_BACK_WHITE),
+            ('bggray', ANSI_BACK_WHITE),
+            ('bgdimgray', hilite + ANSI_BACK_BLACK),
+            ('bgblack', ANSI_BACK_BLACK),
+    ]
+
+    # make sure to escape [
+    colorcodes = [(c, code.replace("[", r"\[")) for c, code in colorcodes]
+    colorback = [(c, code.replace("[", r"\[")) for c, code in colorback]
+    # create stop markers
+    fgstop = [("", c.replace("[", r"\[")) for c in (normal, hilite, underline)]
+    bgstop = [("", c.replace("[", r"\[")) for c in (normal,)]
+    fgstop = "|".join(co[1] for co in colorcodes + fgstop + [("", "$")])
+    bgstop = "|".join(co[1] for co in colorback + bgstop + [("", "$")])
+
+    # pre-compile regexes
+    re_fgs = [(cname, re.compile("(?:%s)(.*?)(?=%s)" % (code, fgstop))) for cname, code in colorcodes]
+    re_bgs = [(cname, re.compile("(?:%s)(.*?)(?=%s)" % (code, bgstop))) for cname, code in colorback]
+    re_normal = re.compile(normal.replace("[", r"\["))
+    re_hilite = re.compile("(?:%s)(.*)(?=%s)" % (hilite.replace("[", r"\["), fgstop))
+    re_uline = re.compile("(?:%s)(.*?)(?=%s)" % (ANSI_UNDERLINE.replace("[", r"\["), fgstop))
+    re_string = re.compile(r'(?P<htmlchars>[<&>])|(?P<space> [ \t]+)|(?P<lineend>\r\n|\r|\n)', re.S|re.M|re.I)
+    re_link =  re.compile(r'\{lc(.*?)\{lt(.*?)\{le', re.DOTALL)
+
+    def re_color(self, text):
+        """
+        Replace ansi colors with html color class names.
+        Let the client choose how it will display colors, if it wishes to.  """
+        for colorname, regex in self.re_fgs:
+            text = regex.sub(r'''<span class="%s">\1</span>''' % colorname, text)
+        for bgname, regex in self.re_bgs:
+            text = regex.sub(r'''<span class="%s">\1</span>''' % bgname, text)
+        return self.re_normal.sub("", text)
+
+    def re_bold(self, text):
+        "Clean out superfluous hilights rather than set <strong>to make it match the look of telnet."
+        return self.re_hilite.sub(r'<strong>\1</strong>', text)
+
+    def re_underline(self, text):
+        "Replace ansi underline with html underline class name."
+        return self.re_uline.sub(r'<span class="underline">\1</span>', text)
+
+    def remove_bells(self, text):
+        "Remove ansi specials"
+        return text.replace('\07', '')
+
+    def remove_backspaces(self, text):
+        "Removes special escape sequences"
+        backspace_or_eol = r'(.\010)|(\033\[K)'
+        n = 1
+        while n > 0:
+            text, n = re.subn(backspace_or_eol, '', text, 1)
+        return text
+
+    def convert_linebreaks(self, text):
+        "Extra method for cleaning linebreaks"
+        return text.replace(r'\n', r'<br>')
+
+    def convert_urls(self, text):
+        "Replace urls (http://...) by valid HTML"
+        regexp = r"((ftp|www|http)(\W+\S+[^).,:;?\]\}(\<span\>) \r\n$\"\']+))"
+        # -> added target to output prevent the web browser from attempting to
+        # change pages (and losing our webclient session).
+        return re.sub(regexp, r'<a href="\1" target="_blank">\1</a>', text)
+
+    def convert_links(self, text):
+        """
+        Replaces links with HTML code
+        """
+        html = "<a href='#' onclick='websocket.send(\"\\1\"); return false;'>\\2</a>"
+        return self.re_link.sub(html, text)
+
+    def do_sub(self, m):
+        "Helper method to be passed to re.sub."
+        c = m.groupdict()
+        if c['htmlchars']:
+            return cgi.escape(c['htmlchars'])
+        if c['lineend']:
+            return '<br>'
+        elif c['space'] == '\t':
+            return ' ' * self.tabstop
+        elif c['space']:
+            t = m.group().replace('\t', '&nbsp;' * self.tabstop)
+            t = t.replace(' ', '&nbsp;')
+            return t
+
+    def parse(self, text, strip_ansi=False):
+        """
+        Main access function, converts a text containing
+        ansi codes into html statements.
+        """
+        # parse everything to ansi first
+        text = parse_ansi(text, strip_ansi=strip_ansi, xterm256=False, mxp=True)
+        # convert all ansi to html
+        result = re.sub(self.re_string, self.do_sub, text)
+        result = self.re_color(result)
+        result = self.re_bold(result)
+        result = self.re_underline(result)
+        result = self.remove_bells(result)
+        result = self.convert_linebreaks(result)
+        result = self.remove_backspaces(result)
+        result = self.convert_urls(result)
+        result = self.convert_links(result)
+        # clean out eventual ansi that was missed
+        #result = parse_ansi(result, strip_ansi=True)
+
+        return result
+
+HTML_PARSER = TextToHTMLparser()
+
+
+#
+# Access function
+#
+
+def parse_html(string, strip_ansi=False, parser=HTML_PARSER):
+    """
+    Parses a string, replace ansi markup with html
+    """
+    return parser.parse(string, strip_ansi=strip_ansi)

lib/utils/txws.py

@@ -0,0 +1,643 @@
+# Copyright (c) 2011 Oregon State University Open Source Lab
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included
+#    in all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+#    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+#    NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+#    DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+#    OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+#    USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Blind reimplementation of WebSockets as a standalone wrapper for Twisted
+protocols.
+"""
+
+__version__ = "0.7.1"
+
+from base64 import b64encode, b64decode
+from hashlib import md5, sha1
+from string import digits
+from struct import pack, unpack
+
+from twisted.internet.interfaces import ISSLTransport
+from twisted.protocols.policies import ProtocolWrapper, WrappingFactory
+from twisted.python import log
+from twisted.web.http import datetimeToString
+
+class WSException(Exception):
+    """
+    Something stupid happened here.
+
+    If this class escapes txWS, then something stupid happened in multiple
+    places.
+    """
+
+# Flavors of WS supported here.
+# HYBI00  - Hixie-76, HyBi-00. Challenge/response after headers, very minimal
+#           framing. Tricky to start up, but very smooth sailing afterwards.
+# HYBI07  - HyBi-07. Modern "standard" handshake. Bizarre masked frames, lots
+#           of binary data packing.
+# HYBI10  - HyBi-10. Just like HyBi-07. No, seriously. *Exactly* the same,
+#           except for the protocol number.
+# RFC6455 - RFC 6455. The official WebSocket protocol standard. The protocol
+#           number is 13, but otherwise it is identical to HyBi-07.
+
+HYBI00, HYBI07, HYBI10, RFC6455 = range(4)
+
+# States of the state machine. Because there are no reliable byte counts for
+# any of this, we don't use StatefulProtocol; instead, we use custom state
+# enumerations. Yay!
+
+REQUEST, NEGOTIATING, CHALLENGE, FRAMES = range(4)
+
+# Control frame specifiers. Some versions of WS have control signals sent
+# in-band. Adorable, right?
+
+NORMAL, CLOSE, PING, PONG = range(4)
+
+opcode_types = {
+    0x0: NORMAL,
+    0x1: NORMAL,
+    0x2: NORMAL,
+    0x8: CLOSE,
+    0x9: PING,
+    0xa: PONG,
+}
+
+encoders = {
+    "base64": b64encode,
+}
+
+decoders = {
+    "base64": b64decode,
+}
+
+# Fake HTTP stuff, and a couple convenience methods for examining fake HTTP
+# headers.
+
+def http_headers(s):
+    """
+    Create a dictionary of data from raw HTTP headers.
+    """
+
+    d = {}
+
+    for line in s.split("\r\n"):
+        try:
+            key, value = [i.strip() for i in line.split(":", 1)]
+            d[key] = value
+        except ValueError:
+            pass
+
+    return d
+
+def is_websocket(headers):
+    """
+    Determine whether a given set of headers is asking for WebSockets.
+    """
+
+    return ("upgrade" in headers.get("Connection", "").lower()
+            and headers.get("Upgrade").lower() == "websocket")
+
+def is_hybi00(headers):
+    """
+    Determine whether a given set of headers is HyBi-00-compliant.
+
+    Hixie-76 and HyBi-00 use a pair of keys in the headers to handshake with
+    servers.
+    """
+
+    return "Sec-WebSocket-Key1" in headers and "Sec-WebSocket-Key2" in headers
+
+# Authentication for WS.
+
+def complete_hybi00(headers, challenge):
+    """
+    Generate the response for a HyBi-00 challenge.
+    """
+
+    key1 = headers["Sec-WebSocket-Key1"]
+    key2 = headers["Sec-WebSocket-Key2"]
+
+    first = int("".join(i for i in key1 if i in digits)) / key1.count(" ")
+    second = int("".join(i for i in key2 if i in digits)) / key2.count(" ")
+
+    nonce = pack(">II8s", first, second, challenge)
+
+    return md5(nonce).digest()
+
+def make_accept(key):
+    """
+    Create an "accept" response for a given key.
+
+    This dance is expected to somehow magically make WebSockets secure.
+    """
+
+    guid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+
+    return sha1("%s%s" % (key, guid)).digest().encode("base64").strip()
+
+# Frame helpers.
+# Separated out to make unit testing a lot easier.
+# Frames are bonghits in newer WS versions, so helpers are appreciated.
+
+def make_hybi00_frame(buf):
+    """
+    Make a HyBi-00 frame from some data.
+
+    This function does exactly zero checks to make sure that the data is safe
+    and valid text without any 0xff bytes.
+    """
+
+    return "\x00%s\xff" % buf
+
+def parse_hybi00_frames(buf):
+    """
+    Parse HyBi-00 frames, returning unwrapped frames and any unmatched data.
+
+    This function does not care about garbage data on the wire between frames,
+    and will actively ignore it.
+    """
+
+    start = buf.find("\x00")
+    tail = 0
+    frames = []
+
+    while start != -1:
+        end = buf.find("\xff", start + 1)
+        if end == -1:
+            # Incomplete frame, try again later.
+            break
+        else:
+            # Found a frame, put it in the list.
+            frame = buf[start + 1:end]
+            frames.append((NORMAL, frame))
+            tail = end + 1
+        start = buf.find("\x00", end + 1)
+
+    # Adjust the buffer and return.
+    buf = buf[tail:]
+    return frames, buf
+
+def mask(buf, key):
+    """
+    Mask or unmask a buffer of bytes with a masking key.
+
+    The key must be exactly four bytes long.
+    """
+
+    # This is super-secure, I promise~
+    key = [ord(i) for i in key]
+    buf = list(buf)
+    for i, char in enumerate(buf):
+        buf[i] = chr(ord(char) ^ key[i % 4])
+    return "".join(buf)
+
+def make_hybi07_frame(buf, opcode=0x1):
+    """
+    Make a HyBi-07 frame.
+
+    This function always creates unmasked frames, and attempts to use the
+    smallest possible lengths.
+    """
+
+    if len(buf) > 0xffff:
+        length = "\x7f%s" % pack(">Q", len(buf))
+    elif len(buf) > 0x7d:
+        length = "\x7e%s" % pack(">H", len(buf))
+    else:
+        length = chr(len(buf))
+
+    # Always make a normal packet.
+    header = chr(0x80 | opcode)
+    frame = "%s%s%s" % (header, length, buf)
+    return frame
+
+def make_hybi07_frame_dwim(buf):
+    """
+    Make a HyBi-07 frame with binary or text data according to the type of buf.
+    """
+    
+    # TODO: eliminate magic numbers.
+    if isinstance(buf, str):
+        return make_hybi07_frame(buf, opcode=0x2)
+    elif isinstance(buf, unicode):
+        return make_hybi07_frame(buf.encode("utf-8"), opcode=0x1)
+    else:
+        raise TypeError("In binary support mode, frame data must be either str or unicode")
+
+def parse_hybi07_frames(buf):
+    """
+    Parse HyBi-07 frames in a highly compliant manner.
+    """
+
+    start = 0
+    frames = []
+
+    while True:
+        # If there's not at least two bytes in the buffer, bail.
+        if len(buf) - start < 2:
+            break
+
+        # Grab the header. This single byte holds some flags nobody cares
+        # about, and an opcode which nobody cares about.
+        header = ord(buf[start])
+        if header & 0x70:
+            # At least one of the reserved flags is set. Pork chop sandwiches!
+            raise WSException("Reserved flag in HyBi-07 frame (%d)" % header)
+            frames.append(("", CLOSE))
+            return frames, buf
+
+        # Get the opcode, and translate it to a local enum which we actually
+        # care about.
+        opcode = header & 0xf
+        try:
+            opcode = opcode_types[opcode]
+        except KeyError:
+            raise WSException("Unknown opcode %d in HyBi-07 frame" % opcode)
+
+        # Get the payload length and determine whether we need to look for an
+        # extra length.
+        length = ord(buf[start + 1])
+        masked = length & 0x80
+        length &= 0x7f
+
+        # The offset we're gonna be using to walk through the frame. We use
+        # this because the offset is variable depending on the length and
+        # mask.
+        offset = 2
+
+        # Extra length fields.
+        if length == 0x7e:
+            if len(buf) - start < 4:
+                break
+
+            length = buf[start + 2:start + 4]
+            length = unpack(">H", length)[0]
+            offset += 2
+        elif length == 0x7f:
+            if len(buf) - start < 10:
+                break
+
+            # Protocol bug: The top bit of this long long *must* be cleared;
+            # that is, it is expected to be interpreted as signed. That's
+            # fucking stupid, if you don't mind me saying so, and so we're
+            # interpreting it as unsigned anyway. If you wanna send exabytes
+            # of data down the wire, then go ahead!
+            length = buf[start + 2:start + 10]
+            length = unpack(">Q", length)[0]
+            offset += 8
+
+        if masked:
+            if len(buf) - (start + offset) < 4:
+                break
+
+            key = buf[start + offset:start + offset + 4]
+            offset += 4
+
+        if len(buf) - (start + offset) < length:
+            break
+
+        data = buf[start + offset:start + offset + length]
+
+        if masked:
+            data = mask(data, key)
+
+        if opcode == CLOSE:
+            if len(data) >= 2:
+                # Gotta unpack the opcode and return usable data here.
+                data = unpack(">H", data[:2])[0], data[2:]
+            else:
+                # No reason given; use generic data.
+                data = 1000, "No reason given"
+
+        frames.append((opcode, data))
+        start += offset + length
+
+    return frames, buf[start:]
+
+class WebSocketProtocol(ProtocolWrapper):
+    """
+    Protocol which wraps another protocol to provide a WebSockets transport
+    layer.
+    """
+
+    buf = ""
+    codec = None
+    location = "/"
+    host = "example.com"
+    origin = "http://example.com"
+    state = REQUEST
+    flavor = None
+    do_binary_frames = False
+
+    def __init__(self, *args, **kwargs):
+        ProtocolWrapper.__init__(self, *args, **kwargs)
+        self.pending_frames = []
+
+    def setBinaryMode(self, mode):
+        """
+        If True, send str as binary and unicode as text.
+        
+        Defaults to false for backwards compatibility.
+        """
+        self.do_binary_frames = bool(mode)
+
+    def isSecure(self):
+        """
+        Borrowed technique for determining whether this connection is over
+        SSL/TLS.
+        """
+
+        return ISSLTransport(self.transport, None) is not None
+
+    def sendCommonPreamble(self):
+        """
+        Send the preamble common to all WebSockets connections.
+
+        This might go away in the future if WebSockets continue to diverge.
+        """
+
+        self.transport.writeSequence([
+            "HTTP/1.1 101 FYI I am not a webserver\r\n",
+            "Server: TwistedWebSocketWrapper/1.0\r\n",
+            "Date: %s\r\n" % datetimeToString(),
+            "Upgrade: WebSocket\r\n",
+            "Connection: Upgrade\r\n",
+        ])
+
+    def sendHyBi00Preamble(self):
+        """
+        Send a HyBi-00 preamble.
+        """
+
+        protocol = "wss" if self.isSecure() else "ws"
+
+        self.sendCommonPreamble()
+
+        self.transport.writeSequence([
+            "Sec-WebSocket-Origin: %s\r\n" % self.origin,
+            "Sec-WebSocket-Location: %s://%s%s\r\n" % (protocol, self.host,
+                                                       self.location),
+            "WebSocket-Protocol: %s\r\n" % self.codec,
+            "Sec-WebSocket-Protocol: %s\r\n" % self.codec,
+            "\r\n",
+        ])
+
+    def sendHyBi07Preamble(self):
+        """
+        Send a HyBi-07 preamble.
+        """
+
+        self.sendCommonPreamble()
+        challenge = self.headers["Sec-WebSocket-Key"]
+        response = make_accept(challenge)
+
+        self.transport.write("Sec-WebSocket-Accept: %s\r\n\r\n" % response)
+
+    def parseFrames(self):
+        """
+        Find frames in incoming data and pass them to the underlying protocol.
+        """
+
+        if self.flavor == HYBI00:
+            parser = parse_hybi00_frames
+        elif self.flavor in (HYBI07, HYBI10, RFC6455):
+            parser = parse_hybi07_frames
+        else:
+            raise WSException("Unknown flavor %r" % self.flavor)
+
+        try:
+            frames, self.buf = parser(self.buf)
+        except WSException, wse:
+            # Couldn't parse all the frames, something went wrong, let's bail.
+            self.close(wse.args[0])
+            return
+
+        for frame in frames:
+            opcode, data = frame
+            if opcode == NORMAL:
+                # Business as usual. Decode the frame, if we have a decoder.
+                if self.codec:
+                    data = decoders[self.codec](data)
+                # Pass the frame to the underlying protocol.
+                ProtocolWrapper.dataReceived(self, data)
+            elif opcode == CLOSE:
+                # The other side wants us to close. I wonder why?
+                reason, text = data
+                log.msg("Closing connection: %r (%d)" % (text, reason))
+
+                # Close the connection.
+                self.close()
+
+    def sendFrames(self):
+        """
+        Send all pending frames.
+        """
+
+        if self.state != FRAMES:
+            return
+
+        if self.flavor == HYBI00:
+            maker = make_hybi00_frame
+        elif self.flavor in (HYBI07, HYBI10, RFC6455):
+            if self.do_binary_frames:
+                maker = make_hybi07_frame_dwim
+            else:
+                maker = make_hybi07_frame
+        else:
+            raise WSException("Unknown flavor %r" % self.flavor)
+
+        for frame in self.pending_frames:
+            # Encode the frame before sending it.
+            if self.codec:
+                frame = encoders[self.codec](frame)
+            packet = maker(frame)
+            self.transport.write(packet)
+        self.pending_frames = []
+
+    def validateHeaders(self):
+        """
+        Check received headers for sanity and correctness, and stash any data
+        from them which will be required later.
+        """
+
+        # Obvious but necessary.
+        if not is_websocket(self.headers):
+            log.msg("Not handling non-WS request")
+            return False
+
+        # Stash host and origin for those browsers that care about it.
+        if "Host" in self.headers:
+            self.host = self.headers["Host"]
+        if "Origin" in self.headers:
+            self.origin = self.headers["Origin"]
+
+        # Check whether a codec is needed. WS calls this a "protocol" for
+        # reasons I cannot fathom. Newer versions of noVNC (0.4+) sets
+        # multiple comma-separated codecs, handle this by chosing first one
+        # we can encode/decode.
+        protocols = None
+        if "WebSocket-Protocol" in self.headers:
+            protocols = self.headers["WebSocket-Protocol"]
+        elif "Sec-WebSocket-Protocol" in self.headers:
+            protocols = self.headers["Sec-WebSocket-Protocol"]
+
+        if isinstance(protocols, basestring):
+            protocols = [p.strip() for p in protocols.split(',')]
+
+            for protocol in protocols:
+                if protocol in encoders or protocol in decoders:
+                    log.msg("Using WS protocol %s!" % protocol)
+                    self.codec = protocol
+                    break
+
+                log.msg("Couldn't handle WS protocol %s!" % protocol)
+
+            if not self.codec:
+                return False
+
+        # Start the next phase of the handshake for HyBi-00.
+        if is_hybi00(self.headers):
+            log.msg("Starting HyBi-00/Hixie-76 handshake")
+            self.flavor = HYBI00
+            self.state = CHALLENGE
+
+        # Start the next phase of the handshake for HyBi-07+.
+        if "Sec-WebSocket-Version" in self.headers:
+            version = self.headers["Sec-WebSocket-Version"]
+            if version == "7":
+                log.msg("Starting HyBi-07 conversation")
+                self.sendHyBi07Preamble()
+                self.flavor = HYBI07
+                self.state = FRAMES
+            elif version == "8":
+                log.msg("Starting HyBi-10 conversation")
+                self.sendHyBi07Preamble()
+                self.flavor = HYBI10
+                self.state = FRAMES
+            elif version == "13":
+                log.msg("Starting RFC 6455 conversation")
+                self.sendHyBi07Preamble()
+                self.flavor = RFC6455
+                self.state = FRAMES
+            else:
+                log.msg("Can't support protocol version %s!" % version)
+                return False
+
+        return True
+
+    def dataReceived(self, data):
+        self.buf += data
+
+        oldstate = None
+
+        while oldstate != self.state:
+            oldstate = self.state
+
+            # Handle initial requests. These look very much like HTTP
+            # requests, but aren't. We need to capture the request path for
+            # those browsers which want us to echo it back to them (Chrome,
+            # mainly.)
+            # These lines look like:
+            # GET /some/path/to/a/websocket/resource HTTP/1.1
+            if self.state == REQUEST:
+                if "\r\n" in self.buf:
+                    request, chaff, self.buf = self.buf.partition("\r\n")
+                    try:
+                        verb, self.location, version = request.split(" ")
+                    except ValueError:
+                        self.loseConnection()
+                    else:
+                        self.state = NEGOTIATING
+
+            elif self.state == NEGOTIATING:
+                # Check to see if we've got a complete set of headers yet.
+                if "\r\n\r\n" in self.buf:
+                    head, chaff, self.buf = self.buf.partition("\r\n\r\n")
+                    self.headers = http_headers(head)
+                    # Validate headers. This will cause a state change.
+                    if not self.validateHeaders():
+                        self.loseConnection()
+
+            elif self.state == CHALLENGE:
+                # Handle the challenge. This is completely exclusive to
+                # HyBi-00/Hixie-76.
+                if len(self.buf) >= 8:
+                    challenge, self.buf = self.buf[:8], self.buf[8:]
+                    response = complete_hybi00(self.headers, challenge)
+                    self.sendHyBi00Preamble()
+                    self.transport.write(response)
+                    log.msg("Completed HyBi-00/Hixie-76 handshake")
+                    # We're all finished here; start sending frames.
+                    self.state = FRAMES
+
+            elif self.state == FRAMES:
+                self.parseFrames()
+
+        # Kick any pending frames. This is needed because frames might have
+        # started piling up early; we can get write()s from our protocol above
+        # when they makeConnection() immediately, before our browser client
+        # actually sends any data. In those cases, we need to manually kick
+        # pending frames.
+        if self.pending_frames:
+            self.sendFrames()
+
+    def write(self, data):
+        """
+        Write to the transport.
+
+        This method will only be called by the underlying protocol.
+        """
+
+        self.pending_frames.append(data)
+        self.sendFrames()
+
+    def writeSequence(self, data):
+        """
+        Write a sequence of data to the transport.
+
+        This method will only be called by the underlying protocol.
+        """
+
+        self.pending_frames.extend(data)
+        self.sendFrames()
+
+    def close(self, reason=""):
+        """
+        Close the connection.
+
+        This includes telling the other side we're closing the connection.
+
+        If the other side didn't signal that the connection is being closed,
+        then we might not see their last message, but since their last message
+        should, according to the spec, be a simple acknowledgement, it
+        shouldn't be a problem.
+        """
+
+        # Send a closing frame. It's only polite. (And might keep the browser
+        # from hanging.)
+        if self.flavor in (HYBI07, HYBI10, RFC6455):
+            frame = make_hybi07_frame(reason, opcode=0x8)
+            self.transport.write(frame)
+
+        self.loseConnection()
+
+class WebSocketFactory(WrappingFactory):
+    """
+    Factory which wraps another factory to provide WebSockets transports for
+    all of its protocols.
+    """
+
+    protocol = WebSocketProtocol