Adding full convertion to the migrations, should hopefully cover all cases now.

src/objects/migrations/0009_converting_attributes.py

@@ -9,7 +9,143 @@ try:
 except ImportError:
     import pickle
 from src.utils.utils import to_str, to_unicode
-from src.typeclasses.models import PackedDBobject
+from src.typeclasses.models import PackedDBobject,PackedDict,PackedList
+
+from django.contrib.contenttypes.models import ContentType
+CTYPEGET = ContentType.objects.get
+GA = object.__getattribute__
+SA = object.__setattr__
+DA = object.__delattr__
+
+def to_attr(data):
+    """
+    Convert data to proper attr data format before saving
+
+    We have to make sure to not store database objects raw, since
+    this will crash the system. Instead we must store their IDs
+    and make sure to convert back when the attribute is read back
+    later.
+
+    Due to this it's criticial that we check all iterables
+    recursively, converting all found database objects to a form
+    the database can handle. We handle lists, tuples and dicts
+    (and any nested combination of them) this way, all other
+    iterables are stored and returned as lists.
+
+    data storage format: 
+       (simple|dbobj|iter, <data>)
+    where 
+       simple - a single non-db object, like a string or number
+       dbobj - a single dbobj
+       iter - any iterable object - will be looped over recursively
+              to convert dbobj->id. 
+
+    """
+
+    def iter_db2id(item):
+        """
+        recursively looping through stored iterables, replacing objects with ids.
+        (Python only builds nested functions once, so there is no overhead for nesting)
+        """
+        dtype = type(item)
+        if dtype in (basestring, int, float): # check the most common types first, for speed
+            return item 
+        elif hasattr(item, "id") and hasattr(item, "db_model_name") and hasattr(item, "db_key"):
+            db_model_name = item.db_model_name
+            if db_model_name == "typeclass":
+                db_model_name = GA(item.dbobj, "db_model_name")
+            return PackedDBobject(item.id, db_model_name, item.db_key)
+        elif dtype == tuple:
+            return tuple(iter_db2id(val) for val in item)
+        elif dtype in (dict, PackedDict):
+            return dict((key, iter_db2id(val)) for key, val in item.items())
+        elif hasattr(item, '__iter__'):
+            return list(iter_db2id(val) for val in item)
+        else:
+            return item
+
+    dtype = type(data)
+
+    if dtype in (basestring, int, float):
+        return ("simple",data)
+    elif hasattr(data, "id") and hasattr(data, "db_model_name") and hasattr(data, 'db_key'):
+        # all django models (objectdb,scriptdb,playerdb,channel,msg,typeclass)
+        # have the protected property db_model_name hardcoded on themselves for speed.
+        db_model_name = data.db_model_name
+        if db_model_name == "typeclass":
+            # typeclass cannot help us, we want the actual child object model name
+            db_model_name = GA(data.dbobj, "db_model_name")
+        return ("dbobj", PackedDBobject(data.id, db_model_name, data.db_key))        
+    elif hasattr(data, "__iter__"): 
+        return ("iter", iter_db2id(data))
+    else:
+        return ("simple", data)
+
+def from_attr(attr, datatuple):
+    """
+    Retrieve data from a previously stored attribute. This
+    is always a dict with keys type and data.                 
+
+    datatuple comes from the database storage and has 
+    the following format: 
+       (simple|dbobj|iter, <data>)
+    where
+        simple - a single non-db object, like a string. is returned as-is.
+        dbobj - a single dbobj-id. This id is retrieved back from the database. 
+        iter - an iterable. This is traversed iteratively, converting all found
+               dbobj-ids back to objects. Also, all lists and dictionaries are 
+               returned as their PackedList/PackedDict counterparts in order to 
+               allow in-place assignment such as obj.db.mylist[3] = val. Mylist
+               is then a PackedList that saves the data on the fly. 
+    """
+    # nested functions 
+    def id2db(data):
+        """
+        Convert db-stored dbref back to object
+        """
+        mclass = CTYPEGET(model=data.db_model).model_class()
+        try:
+            return mclass.objects.dbref_search(data.id)
+
+        except AttributeError:
+            try:
+                return mclass.objects.get(id=data.id)
+            except mclass.DoesNotExist: # could happen if object was deleted in the interim.
+                return None                
+
+    def iter_id2db(item):
+        """
+        Recursively looping through stored iterables, replacing ids with actual objects.
+        We return PackedDict and PackedLists instead of normal lists; this is needed in order for
+        the user to do dynamic saving of nested in-place, such as obj.db.attrlist[2]=3. What is
+        stored in the database are however always normal python primitives. 
+        """
+        dtype = type(item)
+        if dtype in (basestring, int, float): # check the most common types first, for speed
+            return item 
+        elif dtype == PackedDBobject:
+            return id2db(item)
+        elif dtype == tuple:                        
+            return tuple([iter_id2db(val) for val in item])
+        elif dtype in (dict, PackedDict):
+            return PackedDict(attr, dict(zip([key for key in item.keys()],
+                                             [iter_id2db(val) for val in item.values()])))
+        elif hasattr(item, '__iter__'):
+            return PackedList(attr, list(iter_id2db(val) for val in item))
+        else: 
+            return item 
+
+    typ, data = datatuple
+
+    if typ == 'simple': 
+        # single non-db objects
+        return data
+    elif typ == 'dbobj': 
+        # a single stored dbobj        
+        return id2db(data)
+    elif typ == 'iter': 
+        # all types of iterables
+        return iter_id2db(data)
 
 class Migration(DataMigration):
 
@@ -24,7 +160,8 @@ class Migration(DataMigration):
                 val = ("dbobj", val)
             else:
                 val = ("simple", val)
-            attr.value = attr.from_attr(val)
+            attr.db_value = to_unicode(pickle.dumps(to_str(to_attr(from_attr(attr, val)))))
+            attr.save()
                                       
     def backwards(self, orm):
         "Write your backwards methods here."

src/players/migrations/0008_converting_attributes.py

@@ -9,7 +9,145 @@ try:
 except ImportError:
     import pickle
 from src.utils.utils import to_str, to_unicode
-from src.typeclasses.models import PackedDBobject
+from src.typeclasses.models import PackedDBobject,PackedDict,PackedList
+from src.players.models import PlayerAttribute
+
+from django.contrib.contenttypes.models import ContentType
+CTYPEGET = ContentType.objects.get
+GA = object.__getattribute__
+SA = object.__setattr__
+DA = object.__delattr__
+
+def to_attr(data):
+    """
+    Convert data to proper attr data format before saving
+
+    We have to make sure to not store database objects raw, since
+    this will crash the system. Instead we must store their IDs
+    and make sure to convert back when the attribute is read back
+    later.
+
+    Due to this it's criticial that we check all iterables
+    recursively, converting all found database objects to a form
+    the database can handle. We handle lists, tuples and dicts
+    (and any nested combination of them) this way, all other
+    iterables are stored and returned as lists.
+
+    data storage format: 
+       (simple|dbobj|iter, <data>)
+    where 
+       simple - a single non-db object, like a string or number
+       dbobj - a single dbobj
+       iter - any iterable object - will be looped over recursively
+              to convert dbobj->id. 
+
+    """
+
+    def iter_db2id(item):
+        """
+        recursively looping through stored iterables, replacing objects with ids.
+        (Python only builds nested functions once, so there is no overhead for nesting)
+        """
+        dtype = type(item)
+        if dtype in (basestring, int, float): # check the most common types first, for speed
+            return item 
+        elif hasattr(item, "id") and hasattr(item, "db_model_name") and hasattr(item, "db_key"):
+            db_model_name = item.db_model_name
+            if db_model_name == "typeclass":
+                db_model_name = GA(item.dbobj, "db_model_name")
+            return PackedDBobject(item.id, db_model_name, item.db_key)
+        elif dtype == tuple:
+            return tuple(iter_db2id(val) for val in item)
+        elif dtype in (dict, PackedDict):
+            return dict((key, iter_db2id(val)) for key, val in item.items())
+        elif hasattr(item, '__iter__'):
+            return list(iter_db2id(val) for val in item)
+        else:
+            return item
+
+    dtype = type(data)
+
+    if dtype in (basestring, int, float):
+        return ("simple",data)
+    elif hasattr(data, "id") and hasattr(data, "db_model_name") and hasattr(data, 'db_key'):
+        # all django models (objectdb,scriptdb,playerdb,channel,msg,typeclass)
+        # have the protected property db_model_name hardcoded on themselves for speed.
+        db_model_name = data.db_model_name
+        if db_model_name == "typeclass":
+            # typeclass cannot help us, we want the actual child object model name
+            db_model_name = GA(data.dbobj, "db_model_name")
+        return ("dbobj", PackedDBobject(data.id, db_model_name, data.db_key))        
+    elif hasattr(data, "__iter__"): 
+        return ("iter", iter_db2id(data))
+    else:
+        return ("simple", data)
+
+def from_attr(attr, datatuple):
+    """
+    Retrieve data from a previously stored attribute. This
+    is always a dict with keys type and data.                 
+
+    datatuple comes from the database storage and has 
+    the following format: 
+       (simple|dbobj|iter, <data>)
+    where
+        simple - a single non-db object, like a string. is returned as-is.
+        dbobj - a single dbobj-id. This id is retrieved back from the database. 
+        iter - an iterable. This is traversed iteratively, converting all found
+               dbobj-ids back to objects. Also, all lists and dictionaries are 
+               returned as their PackedList/PackedDict counterparts in order to 
+               allow in-place assignment such as obj.db.mylist[3] = val. Mylist
+               is then a PackedList that saves the data on the fly. 
+    """
+    # nested functions 
+    def id2db(data):
+        """
+        Convert db-stored dbref back to object
+        """
+        mclass = CTYPEGET(model=data.db_model).model_class()
+        try:
+            return mclass.objects.dbref_search(data.id)
+
+        except AttributeError:
+            try:
+                return mclass.objects.get(id=data.id)
+            except mclass.DoesNotExist: # could happen if object was deleted in the interim.
+                return None                
+
+    def iter_id2db(item):
+        """
+        Recursively looping through stored iterables, replacing ids with actual objects.
+        We return PackedDict and PackedLists instead of normal lists; this is needed in order for
+        the user to do dynamic saving of nested in-place, such as obj.db.attrlist[2]=3. What is
+        stored in the database are however always normal python primitives. 
+        """
+        dtype = type(item)
+        if dtype in (basestring, int, float): # check the most common types first, for speed
+            return item 
+        elif dtype == PackedDBobject:
+            return id2db(item)
+        elif dtype == tuple:                        
+            return tuple([iter_id2db(val) for val in item])
+        elif dtype in (dict, PackedDict):
+            return PackedDict(attr, dict(zip([key for key in item.keys()],
+                                             [iter_id2db(val) for val in item.values()])))
+        elif hasattr(item, '__iter__'):
+            return PackedList(attr, list(iter_id2db(val) for val in item))
+        else: 
+            return item 
+
+    typ, data = datatuple
+
+    if typ == 'simple': 
+        # single non-db objects
+        return data
+    elif typ == 'dbobj': 
+        # a single stored dbobj        
+        return id2db(data)
+    elif typ == 'iter': 
+        # all types of iterables
+        return iter_id2db(data)
+
 
 class Migration(DataMigration):
 
@@ -24,7 +162,8 @@ class Migration(DataMigration):
                 val = ("dbobj", val)
             else:
                 val = ("simple", val)
-            attr.value = attr.from_attr(val)
+            attr.db_value = to_unicode(pickle.dumps(to_str(to_attr(from_attr(attr, val)))))
+            attr.save()
 
     def backwards(self, orm):
         "Write your backwards methods here."

src/scripts/migrations/0005_converting_attributes.py

@@ -9,7 +9,143 @@ try:
 except ImportError:
     import pickle
 from src.utils.utils import to_str, to_unicode
-from src.typeclasses.models import PackedDBobject
+from src.typeclasses.models import PackedDBobject,PackedDict,PackedList
+
+from django.contrib.contenttypes.models import ContentType
+CTYPEGET = ContentType.objects.get
+GA = object.__getattribute__
+SA = object.__setattr__
+DA = object.__delattr__
+
+def to_attr(data):
+    """
+    Convert data to proper attr data format before saving
+
+    We have to make sure to not store database objects raw, since
+    this will crash the system. Instead we must store their IDs
+    and make sure to convert back when the attribute is read back
+    later.
+
+    Due to this it's criticial that we check all iterables
+    recursively, converting all found database objects to a form
+    the database can handle. We handle lists, tuples and dicts
+    (and any nested combination of them) this way, all other
+    iterables are stored and returned as lists.
+
+    data storage format: 
+       (simple|dbobj|iter, <data>)
+    where 
+       simple - a single non-db object, like a string or number
+       dbobj - a single dbobj
+       iter - any iterable object - will be looped over recursively
+              to convert dbobj->id. 
+
+    """
+
+    def iter_db2id(item):
+        """
+        recursively looping through stored iterables, replacing objects with ids.
+        (Python only builds nested functions once, so there is no overhead for nesting)
+        """
+        dtype = type(item)
+        if dtype in (basestring, int, float): # check the most common types first, for speed
+            return item 
+        elif hasattr(item, "id") and hasattr(item, "db_model_name") and hasattr(item, "db_key"):
+            db_model_name = item.db_model_name
+            if db_model_name == "typeclass":
+                db_model_name = GA(item.dbobj, "db_model_name")
+            return PackedDBobject(item.id, db_model_name, item.db_key)
+        elif dtype == tuple:
+            return tuple(iter_db2id(val) for val in item)
+        elif dtype in (dict, PackedDict):
+            return dict((key, iter_db2id(val)) for key, val in item.items())
+        elif hasattr(item, '__iter__'):
+            return list(iter_db2id(val) for val in item)
+        else:
+            return item
+
+    dtype = type(data)
+
+    if dtype in (basestring, int, float):
+        return ("simple",data)
+    elif hasattr(data, "id") and hasattr(data, "db_model_name") and hasattr(data, 'db_key'):
+        # all django models (objectdb,scriptdb,playerdb,channel,msg,typeclass)
+        # have the protected property db_model_name hardcoded on themselves for speed.
+        db_model_name = data.db_model_name
+        if db_model_name == "typeclass":
+            # typeclass cannot help us, we want the actual child object model name
+            db_model_name = GA(data.dbobj, "db_model_name")
+        return ("dbobj", PackedDBobject(data.id, db_model_name, data.db_key))        
+    elif hasattr(data, "__iter__"): 
+        return ("iter", iter_db2id(data))
+    else:
+        return ("simple", data)
+
+def from_attr(attr, datatuple):
+    """
+    Retrieve data from a previously stored attribute. This
+    is always a dict with keys type and data.                 
+
+    datatuple comes from the database storage and has 
+    the following format: 
+       (simple|dbobj|iter, <data>)
+    where
+        simple - a single non-db object, like a string. is returned as-is.
+        dbobj - a single dbobj-id. This id is retrieved back from the database. 
+        iter - an iterable. This is traversed iteratively, converting all found
+               dbobj-ids back to objects. Also, all lists and dictionaries are 
+               returned as their PackedList/PackedDict counterparts in order to 
+               allow in-place assignment such as obj.db.mylist[3] = val. Mylist
+               is then a PackedList that saves the data on the fly. 
+    """
+    # nested functions 
+    def id2db(data):
+        """
+        Convert db-stored dbref back to object
+        """
+        mclass = CTYPEGET(model=data.db_model).model_class()
+        try:
+            return mclass.objects.dbref_search(data.id)
+
+        except AttributeError:
+            try:
+                return mclass.objects.get(id=data.id)
+            except mclass.DoesNotExist: # could happen if object was deleted in the interim.
+                return None                
+
+    def iter_id2db(item):
+        """
+        Recursively looping through stored iterables, replacing ids with actual objects.
+        We return PackedDict and PackedLists instead of normal lists; this is needed in order for
+        the user to do dynamic saving of nested in-place, such as obj.db.attrlist[2]=3. What is
+        stored in the database are however always normal python primitives. 
+        """
+        dtype = type(item)
+        if dtype in (basestring, int, float): # check the most common types first, for speed
+            return item 
+        elif dtype == PackedDBobject:
+            return id2db(item)
+        elif dtype == tuple:                        
+            return tuple([iter_id2db(val) for val in item])
+        elif dtype in (dict, PackedDict):
+            return PackedDict(attr, dict(zip([key for key in item.keys()],
+                                             [iter_id2db(val) for val in item.values()])))
+        elif hasattr(item, '__iter__'):
+            return PackedList(attr, list(iter_id2db(val) for val in item))
+        else: 
+            return item 
+
+    typ, data = datatuple
+
+    if typ == 'simple': 
+        # single non-db objects
+        return data
+    elif typ == 'dbobj': 
+        # a single stored dbobj        
+        return id2db(data)
+    elif typ == 'iter': 
+        # all types of iterables
+        return iter_id2db(data)
 
 class Migration(DataMigration):
 
@@ -24,7 +160,8 @@ class Migration(DataMigration):
                 val = ("dbobj", val)
             else:
                 val = ("simple", val)
-            attr.value = attr.from_attr(val)
+            attr.db_value = to_unicode(pickle.dumps(to_str(to_attr(from_attr(attr, val)))))
+            attr.save()
  
     def backwards(self, orm):
         "Write your backwards methods here."