Further refactoring.

evennia/server/server.py

@@ -349,9 +349,8 @@ class Evennia(object):
         from evennia.server.models import ServerConfig
         from evennia.utils import gametime as _GAMETIME_MODULE
 
-        if WEBSERVER_ENABLED:
+        # lock the threadpool from accepting more requests
-            # finish all pending web requests. Otherwise stopping threadpool will cause deadlock.
+        self.web_root.pool.lock()
-            yield self.web_root.get_pending_requests()
 
         if mode == 'reload':
             # call restart hooks
@@ -398,11 +397,11 @@ class Evennia(object):
             # flag to avoid loops.
             self.shutdown_complete = True
             if WEBSERVER_ENABLED:
-                # Just to be extra sure, get all pending requests that might have occurred after we started
+                # Make sure to not continue until threadpool queue is empty.
-                d = self.web_root.get_pending_requests()
+                deferred = self.web_root.get_pending_requests()
-                d.addCallback(lambda _: reactor.stop())
+                deferred.addCallback(lambda _: reactor.stop())
                 from twisted.internet import task
-                yield task.deferLater(reactor, 1, d.callback, None)
+                yield task.deferLater(reactor, 1, deferred.callback, None)
             else:
                 # kill the server
                 reactor.callLater(1, reactor.stop)
@@ -538,12 +537,12 @@ if WEBSERVER_ENABLED:
 
     # Start a django-compatible webserver.
 
-    from twisted.python import threadpool
+    #from twisted.python import threadpool
-    from evennia.server.webserver import DjangoWebRoot, WSGIWebServer, Website
+    from evennia.server.webserver import DjangoWebRoot, WSGIWebServer, Website, LockableThreadPool
 
     # start a thread pool and define the root url (/) as a wsgi resource
     # recognized by Django
-    threads = threadpool.ThreadPool(minthreads=max(1, settings.WEBSERVER_THREADPOOL_LIMITS[0]),
+    threads = LockableThreadPool(minthreads=max(1, settings.WEBSERVER_THREADPOOL_LIMITS[0]),
                                     maxthreads=max(1, settings.WEBSERVER_THREADPOOL_LIMITS[1]))
 
     web_root = DjangoWebRoot(threads)

evennia/server/webserver.py

@@ -18,6 +18,8 @@ from twisted.internet import reactor
 from twisted.application import internet
 from twisted.web.proxy import ReverseProxyResource
 from twisted.web.server import NOT_DONE_YET
+from twisted.python import threadpool
+from twisted.internet import defer
 
 from twisted.web.wsgi import WSGIResource
 from django.conf import settings
@@ -29,6 +31,26 @@ _UPSTREAM_IPS = settings.UPSTREAM_IPS
 _DEBUG = settings.DEBUG
 
 
+class LockableThreadPool(threadpool.ThreadPool):
+    """
+    Threadpool that can be locked from accepting new requests.
+    """
+    def __init__(self, *args, **kwargs):
+        self._accept_new = True
+        threadpool.ThreadPool.__init__(self, *args, **kwargs)
+
+    def lock(self):
+        self._accept_new = False
+
+    def callInThread(self, func, *args, **kwargs):
+        """
+        called in the main reactor thread. Makes sure the pool
+        is not locked before continuing.
+        """
+        if self._accept_new:
+            threadpool.ThreadPool.callInThread(self, func, *args, **kwargs)
+
+
 #
 # X-Forwarded-For Handler
 #
@@ -116,9 +138,8 @@ class DjangoWebRoot(resource.Resource):
     """
     This creates a web root (/) that Django
     understands by tweaking the way
-    child instancee ars recognized.
+    child instances are recognized.
     """
-    open_requests = []
 
     def __init__(self, pool):
         """
@@ -128,23 +149,23 @@ class DjangoWebRoot(resource.Resource):
             pool (ThreadPool): The twisted threadpool.
 
         """
+        self._pool = pool
+        self._pending_requests = {}
         resource.Resource.__init__(self)
         self.wsgi_resource = WSGIResource(reactor, pool, WSGIHandler())
 
     def get_pending_requests(self):
         """
-        Converts our open_requests list of deferreds into a DeferredList
+        Converts our _pending_requests list of deferreds into a DeferredList
 
         Returns:
-            d_list (deferred): A DeferredList object of all our requests
+            deflist (DeferredList): Contains all deferreds of pending requests.
+
         """
-        from twisted.internet import defer
+        return defer.DeferredList(self._pending_requests, consumeErrors=True)
-        return defer.DeferredList(self.open_requests, consumeErrors=True)
 
     def _decrement_requests(self, *args, **kwargs):
-        deferred = kwargs.get('deferred', None)
+        self._pending_requests.pop(kwargs.get('deferred', None), None)
-        if deferred in self.open_requests:
-            self.open_requests.remove(deferred)
 
     def getChild(self, path, request):
         """
@@ -155,12 +176,19 @@ class DjangoWebRoot(resource.Resource):
             path (str): Url path.
             request (Request object): Incoming request.
 
+        Notes:
+            We make sure to save the request queue so
+            that we can safely kill the threadpool
+            on a server reload.
+
         """
         path0 = request.prepath.pop(0)
         request.postpath.insert(0, path0)
+
         deferred = request.notifyFinish()
-        self.open_requests.append(deferred)
+        self._pending_requests[deferred] = deferred
         deferred.addBoth(self._decrement_requests, deferred=deferred)
+
         return self.wsgi_resource