UI: fixed "Todo" number for resumeed jobs

arachnado/extensions/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import  

arachnado/extensions/queuesize.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import  
+from scrapy import signals
+
+
+class QueueSizeExtension(object):
+    """
+    This extension adds two fields to scrapy stats:
+
+    * 'scheduler/initial' with an initial number of requests when
+      spider strats;
+    * 'scheduler/remaining' with a remaining number of requests when
+      spider stops.
+
+    """
+    def __init__(self, crawler):
+        crawler.signals.connect(self.spider_opened,
+                                signal=signals.spider_opened)
+        crawler.signals.connect(self.spider_closed,
+                                signal=signals.spider_closed)
+        self.crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def spider_opened(self, spider):
+        self.crawler.stats.set_value('scheduler/initial',
+                                     self._num_requests())
+
+    def spider_closed(self, spider):
+        self.crawler.stats.set_value('scheduler/remaining',
+                                     self._num_requests())
+
+    def _num_requests(self):
+        scheduler = self.crawler.engine.slot.scheduler
+        return len(scheduler)

arachnado/settings.py

@@ -60,6 +60,10 @@
     'arachnado.pipelines.mongoexport.MongoExportPipeline': 10,
 }
 
+EXTENSIONS = {
+    'arachnado.extensions.queuesize.QueueSizeExtension': 100,
+}
+
 MONGO_EXPORT_ENABLED = True
 MONGO_EXPORT_JOBID_KEY = '_job_id'
 HTTPCACHE_ENABLED = False

arachnado/static/js/components/JobList.jsx

@@ -170,6 +170,15 @@ function _formatItemSpeed(info) {
 }
 
 
+/* Return a number of requests in a queue */
+function getNumQueued(stats) {
+    var initialSize = stats['scheduler/initial'] || 0;
+    var enqueued = stats['scheduler/enqueued'] || 0;
+    var dequeued = stats['scheduler/dequeued'] || 0;
+    return initialSize + enqueued - dequeued;
+}
+
+
 function _getRowInfo(job, curTime){
     var stats = job.stats || {};
     var status = simplifiedStatus(job.status);
@@ -205,7 +214,7 @@ function _getRowInfo(job, curTime){
         downloaded: downloaded,
         downloadSpeed: downloadSpeed,
         itemsSpeed: itemsSpeed,
-        todo: (stats['scheduler/enqueued'] || 0) - (stats['scheduler/dequeued'] || 0),
+        todo: getNumQueued(stats),
         shortId: shortId,
         duration: duration
     }