quick read

Author: root

README.md

@@ -38,6 +38,14 @@ SUPERVISORD_PASSWD=something
 
 Run `bash bin/create_db.sh`
 
+
+### Install Quick Read dependencies
+
+1. install cpan
+2. install text::Unidecode in cpan
+3. git clone https://github.com/brucemiller/LaTeXML
+4. perl Makefile.PL; make; make install
+
 ### Fetch resources
 
 Fetch Arxiv papers and tweets.
@@ -62,8 +70,13 @@ PYTHONPATH="." python dlmonitor/webapp/app.py
 bash bin/config_server.sh
 ```
 
-2. Start Gunicorn processes through supervisord
+3. Start Gunicorn processes through supervisord
 
 ```bash
 bash bin/start_supervisord.sh
 ```
+4. Start arxiv source loading worker
+
+```bash
+PYTHONPATH="." python bin/auto_load_arxiv.py --forever
+```

alembic/versions/fb7131fc3951_.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: fb7131fc3951
+Revises: 220426586e09
+Create Date: 2019-06-14 10:51:38.034897
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'fb7131fc3951'
+down_revision = '220426586e09'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('working',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('type', sa.String(length=255), nullable=True),
+    sa.Column('param', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('working')
+    # ### end Alembic commands ###

bin/auto_load_arxiv.py

@@ -0,0 +1,32 @@
+import sys, os
+sys.path.append(".")
+from dlmonitor.db import session_scope
+from dlmonitor.db_models import WorkingQueueModel
+from dlmonitor.latex import build_paper_html, retrieve_paper_html
+from dlmonitor.settings import PDF_PATH
+from sqlalchemy import desc
+from argparse import ArgumentParser
+import logging
+import time
+logging.basicConfig(level=logging.INFO)
+
+if __name__ == '__main__':
+    ap = ArgumentParser()
+    ap.add_argument("--forever", action="store_true")
+    args = ap.parse_args()
+    with session_scope() as session:
+        run_flag = True
+        while run_flag:
+            jobs = session.query(WorkingQueueModel).filter(WorkingQueueModel.type == "load_arxiv").limit(10).all()
+            logging.info("get {} jobs".format(len(jobs)))
+            for job in jobs:
+                arxiv_token = job.param
+                build_paper_html(arxiv_token)
+                logging.info("built {}".format(arxiv_token))
+                session.delete(job)
+            session.commit()
+            if not jobs:
+                time.sleep(3)
+            if not args.forever:
+                # beak
+                run_flag = False

dlmonitor/db_models.py

@@ -58,5 +58,16 @@ class TwitterModel(Base):
     search_vector = Column(TSVectorType('text'))
 
     def __repr__(self):
-        template = '<Arxiv(id="{0}", user_name="{1}")>'
+        template = '<Twitter(id="{0}", user_name="{1}")>'
         return str_repr(template.format(self.id, self.user))
+
+class WorkingQueueModel(Base):
+
+    __tablename__ = "working"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    type = Column(String(255), nullable=True)
+    param = Column(String(255), nullable=True)
+
+    def __repr__(self):
+        return __tablename__ + self.id

dlmonitor/latex.py

@@ -1,12 +1,12 @@
 import urllib2
 import os
+import subprocess
 
 from dlmonitor import settings
 
 def build_paper_html(arxiv_id):
     src_path = "{}/{}".format(settings.SOURCE_PATH, arxiv_id)
     html_path = "{}/main.html".format(src_path)
-    return False
     if os.path.exists(src_path):
         return html_path if os.path.exists(html_path) else None
     opener = urllib2.build_opener()
@@ -17,7 +17,7 @@ def build_paper_html(arxiv_id):
     if (int(file_size) / 1024. / 1024. > 15):
         # File too big
         os.mkdir(src_path)
-        return False
+        return None
     print("download {}: {}".format(arxiv_id, file_size))
     data = page.read()
     os.mkdir(src_path)
@@ -26,23 +26,28 @@ def build_paper_html(arxiv_id):
     os.chdir(src_path)
     os.system("tar xzf {} --directory {}".format(tgz_path, src_path))
     texfiles = [fn for fn in os.listdir(src_path) if fn.endswith(".tex")]
-    select_texfile = texfiles[0]
-    if len(texfiles) > 1:
-        for fn in texfiles:
-            text = open("{}/{}".format(src_path, fn)).read()
-            if "begin{document}" in text:
-                select_texfile = fn
-                break
     if texfiles:
-        os.system("latexml --includestyles --dest=main.xml {}".format(select_texfile.replace(".tex", "")))
+        select_texfile = texfiles[0]
+        if len(texfiles) > 1:
+            for fn in texfiles:
+                text = open("{}/{}".format(src_path, fn)).read()
+                if "begin{document}" in text:
+                    select_texfile = fn
+                    break
+        cmd = "latexml --includestyles --dest=main.xml {}".format(select_texfile.replace(".tex", ""))
+        os.system(cmd)
+        os.system("latexmlpost --dest=main.html main.xml")
         os.system("latexmlpost --dest=main.html main.xml")
     os.remove(tgz_path)
+    open("{}/.loaded".format(src_path), "wb").write("loaded")
     return html_path if os.path.exists(html_path) else None
 
 def retrieve_paper_html(arxiv_token):
     src_path = "{}/{}".format(settings.SOURCE_PATH, arxiv_token)
     html_path = "{}/main.html".format(src_path)
-    if os.path.exists(src_path) and not os.path.exists(html_path):
+    if os.path.exists(src_path) and not os.path.exists("{}/.loaded".format(src_path)):
+        html_body = "PROCESSING"
+    elif os.path.exists(src_path) and not os.path.exists(html_path):
         html_body = "NOT_AVAILABE"
     elif os.path.exists(src_path) and os.path.exists(html_path):
         html_body = open(html_path).read().decode("utf-8")

dlmonitor/webapp/app.py

@@ -1,7 +1,7 @@
 import os
 from flask import Flask, request, redirect, session, send_from_directory
 from flask import render_template, send_from_directory
-from dlmonitor.db import close_global_session
+from dlmonitor.db import close_global_session, get_global_session
 from dlmonitor.fetcher import get_posts
 from dlmonitor import settings
 from urllib2 import unquote
@@ -161,9 +161,19 @@ def save_mendeley():
 
 @app.route("/load_fulltext/<arxiv_token>")
 def load_fulltext(arxiv_token):
-    from dlmonitor.latex import build_paper_html, retrieve_paper_html
-    build_paper_html(arxiv_token)
+    from dlmonitor.db_models import WorkingQueueModel
+    db_session = get_global_session()
+    job = WorkingQueueModel(
+        type="load_arxiv",
+        param=arxiv_token
+    )
+    db_session.add(job)
+    db_session.commit()
+    return "OK"
 
+@app.route("/retrieve_fulltext/<arxiv_token>")
+def retrieve_fulltext(arxiv_token):
+    from dlmonitor.latex import retrieve_paper_html
     return retrieve_paper_html(arxiv_token)
 
 @app.route("/arxiv_files/<arxiv_token>/<path:fp>")

dlmonitor/webapp/static/app.js

@@ -244,14 +244,35 @@ dlmonitor.load_fulltext = function(arxiv_token) {
      success: function(data) {
         // console.log(data);
         dlmonitor.ajaxCount --;
-        if (data == "NOT_AVAILABE") {
+        $("#latex-content").data("arxiv_token", arxiv_token);
+        setTimeout(dlmonitor.retrieve_fulltext, 3000);
+     }
+  });
+}
+
+dlmonitor.retrieve_fulltext = function() {
+  dlmonitor.ajaxCount ++;
+  arxiv_token = $("#latex-content").data("arxiv_token")
+  $.ajax({
+     url: '/retrieve_fulltext/' + arxiv_token,
+     type: 'GET',
+     error: function() {
+          dlmonitor.ajaxCount --;
+          $("#latex-content").html("An error is detected when loading the paper.");
+     },
+     success: function(data) {
+        // console.log(data);
+        dlmonitor.ajaxCount --;
+        if (data == "PROCESSING" || data == "NOT_EXIST") {
+          setTimeout(dlmonitor.retrieve_fulltext, 3000);
+        } else if (data == "NOT_AVAILABE") {
           $("#latex-content").html("This feature is not avaialbe for this paper.");
         } else {
           $("#latex-content").html(data);
         }
      }
   });
-}
+};
 
 dlmonitor.init = function() {
     dlmonitor.updateAll(true);

dlmonitor/webapp/templates/single.html

@@ -7,6 +7,7 @@
     <meta content="Things happening in deep learning: arxiv, twitter, reddit" name="description">
     <link href="https://fonts.googleapis.com/css?family=Source+Sans+Pro:200,300,400,600,700,900" rel="stylesheet">
     <link href="https://bootswatch.com/3/darkly/bootstrap.min.css" rel="stylesheet">
+    <meta content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0' name='viewport' />
     <link href="/static/LaTeXML.css" media="all" rel="stylesheet" type="text/css">
     <link href="/static/ltx-article.css" media="all" rel="stylesheet" type="text/css">
     <link href="/static/default.css?v=15" media="all" rel="stylesheet" type="text/css">
@@ -18,7 +19,7 @@
     </script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/js-cookie/2.1.4/js.cookie.min.js" type="text/javascript">
     </script>
-    <script src="/static/app.js?v=12" type="text/javascript">
+    <script src="/static/app.js?v=13" type="text/javascript">
     </script>
 </head>
 <body>
@@ -54,11 +55,11 @@ <h3><a href="{{post.arxiv_url}}" target="_blank">{{ post.title }}</a></h3>
                             <h3 style="color:#0ce3ac;">Abstract</h3>
                             <p style="font-size: 20px;">{{ post.abstract }}</p>
                             <p>&nbsp;</p>
-                            <h3 style="color:#0ce3ac;">Read Right Now (beta)</h3>
+                            <h3 style="color:#0ce3ac;">Quick Read (beta)</h3>
                             <div id="latex-content">
                               {% if html_body == "NOT_AVAILABE" %}
                               This feature is not avaialbe for this paper.
-                              {% elif html_body == "NOT_EXIST" %}
+                              {% elif html_body == "NOT_EXIST" or html_body == "PROCESSING" %}
                               <div class="sk-folding-cube">
                                 <div class="sk-cube1 sk-cube"></div>
                                 <div class="sk-cube2 sk-cube"></div>
@@ -100,7 +101,7 @@ <h3 style="color:#0ce3ac;">Read Right Now (beta)</h3>
             </div>
         </div>
     </div>
-    {% if html_body == "NOT_EXIST" %}
+    {% if html_body == "NOT_EXIST" or html_body == "PROCESSING" %}
       <script type="text/javascript" defer="defer">
           dlmonitor.load_fulltext("{{ arxiv_token }}");
       </script>