📑Added marker for tests using Github (to easily skip them)

Author: Jean-Francois Thuong

docs/nlp_data.html

@@ -372,11 +372,13 @@ <h1 id="Tests">Tests<a class="anchor-link" href="#Tests"> </a></h1>
         <span class="k">with</span> <span class="n">pytest</span><span class="o">.</span><span class="n">raises</span><span class="p">(</span><span class="ne">FileExistsError</span><span class="p">):</span>
             <span class="n">textual</span> <span class="o">=</span> <span class="n">Textual</span><span class="o">.</span><span class="n">from_path</span><span class="p">(</span><span class="s2">&quot;does_not_exist.txt&quot;</span><span class="p">)</span>
 
+    <span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
     <span class="k">def</span> <span class="nf">test_from_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">check_connection_github</span><span class="p">,</span> <span class="n">local_textual</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Test extract Textual from URL&quot;&quot;&quot;</span>
         <span class="n">textual</span> <span class="o">=</span> <span class="n">Textual</span><span class="o">.</span><span class="n">from_url</span><span class="p">(</span><span class="n">GITHUB_TEST_TXT</span><span class="p">)</span>
         <span class="k">assert</span> <span class="n">textual</span><span class="o">.</span><span class="n">text</span> <span class="o">==</span> <span class="n">local_textual</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;URL text: </span><span class="si">{</span><span class="n">textual</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">&quot;</span>
 
+    <span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
     <span class="k">def</span> <span class="nf">test_from_url_non_ascii</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">check_connection_github</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Test extract Textual from URL with non-ascii characters&quot;&quot;&quot;</span>
         <span class="n">textual</span> <span class="o">=</span> <span class="n">Textual</span><span class="o">.</span><span class="n">from_url</span><span class="p">(</span><span class="n">GITHUB_TEST_TXT_UTF8</span><span class="p">)</span>

docs/nlp_pdf.html

@@ -220,11 +220,13 @@ <h1 id="Tests">Tests<a class="anchor-link" href="#Tests"> </a></h1>
         <span class="n">extra_pages</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">pages_extracted</span><span class="p">)</span> <span class="o">-</span> <span class="nb">set</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">p</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">pages</span><span class="p">)</span>
         <span class="k">assert</span> <span class="n">extra_pages</span> <span class="o">==</span> <span class="nb">set</span><span class="p">(),</span> <span class="sa">f</span><span class="s2">&quot;Extra pages extracted: </span><span class="si">{</span><span class="n">extra_pages</span><span class="si">}</span><span class="s2">&quot;</span>
 
+    <span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
     <span class="k">def</span> <span class="nf">test_from_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">check_connection_github</span><span class="p">,</span> <span class="n">local_textual</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Test extract Textual of PDF from URL&quot;&quot;&quot;</span>
         <span class="n">textual</span> <span class="o">=</span> <span class="n">TextualPDF</span><span class="o">.</span><span class="n">from_url</span><span class="p">(</span><span class="n">GITHUB_TEST_PDF</span><span class="p">)</span>
         <span class="k">assert</span> <span class="n">textual</span><span class="o">.</span><span class="n">text</span> <span class="o">==</span> <span class="n">local_textual</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;URL text: </span><span class="si">{</span><span class="n">textual</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">&quot;</span>
 
+    <span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
     <span class="k">def</span> <span class="nf">test_from_url_pdf</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">check_connection_github</span><span class="p">,</span> <span class="n">local_textual</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;Test extract Textual of PDF from URL using from_url_pdf&quot;&quot;&quot;</span>
         <span class="n">textual</span> <span class="o">=</span> <span class="n">TextualPDF</span><span class="o">.</span><span class="n">from_url_pdf</span><span class="p">(</span><span class="n">GITHUB_TEST_PDF</span><span class="p">)</span>

docs/utils.html

@@ -427,10 +427,10 @@ <h3 id="Try-hushing-various-kinds-of-info">Try hushing various kinds of info<a c
 
 
 
-<div id="7e553850-fe0b-4d5c-98b5-b458b20ad1c7"></div>
+<div id="1e98ee6d-dc2f-45a9-aa69-34230fcf8995"></div>
 <div class="output_subarea output_javascript ">
 <script type="text/javascript">
-var element = $('#7e553850-fe0b-4d5c-98b5-b458b20ad1c7');
+var element = $('#1e98ee6d-dc2f-45a9-aa69-34230fcf8995');
 
     console.info("loading toggle event: 00d9724d-55e4-4ab8-b426-5e92f71f593c")
     const toggle_hush = (e) =>{
@@ -487,10 +487,10 @@ <h3 id="Capable-of-open/close-verbosity-during-the-main-process">Capable of open
 
 
 
-<div id="b44b26f6-5c06-4474-b0ad-f9e5cb0abf04"></div>
+<div id="afbabf0d-95c5-48e2-989d-a92b90992758"></div>
 <div class="output_subarea output_javascript ">
 <script type="text/javascript">
-var element = $('#b44b26f6-5c06-4474-b0ad-f9e5cb0abf04');
+var element = $('#afbabf0d-95c5-48e2-989d-a92b90992758');
 
     console.info("loading toggle event: a5461483-be0b-4a73-8299-458f05a7c0d5")
     const toggle_hush = (e) =>{
@@ -552,10 +552,10 @@ <h3 id="But-we-still-want-the-error-to-be-loud">But we still want the error to b
 
 
 
-<div id="a4a265b4-4017-4d33-b104-1398e100f8d2"></div>
+<div id="639ad8dd-86f0-4d95-a004-d7a16eef7928"></div>
 <div class="output_subarea output_javascript ">
 <script type="text/javascript">
-var element = $('#a4a265b4-4017-4d33-b104-1398e100f8d2');
+var element = $('#639ad8dd-86f0-4d95-a004-d7a16eef7928');
 
     console.info("loading toggle event: 33d84ffd-6fef-48a7-8ddc-26ef5ec8513b")
     const toggle_hush = (e) =>{
@@ -1128,10 +1128,12 @@ <h4 id="url_2_text" class="doc_header"><code>url_2_text</code><a href="https://g
 
 <div class="inner_cell">
     <div class="input_area">
-<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">test_get_url_size</span><span class="p">(</span><span class="n">check_connection_github</span><span class="p">):</span>
+<div class=" highlight hl-ipython3"><pre><span></span><span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
+<span class="k">def</span> <span class="nf">test_get_url_size</span><span class="p">(</span><span class="n">check_connection_github</span><span class="p">):</span>
     <span class="k">assert</span> <span class="n">get_url_size</span><span class="p">(</span><span class="n">url_raw_txt</span><span class="p">)</span> <span class="o">==</span> <span class="mi">264</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;Wrong size for </span><span class="si">{</span><span class="n">url_raw_txt</span><span class="si">}</span><span class="s2">&quot;</span>
 
 
+<span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
 <span class="k">def</span> <span class="nf">test_download_dest</span><span class="p">(</span><span class="n">check_connection_github</span><span class="p">,</span> <span class="n">tmpdir</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;Test download of file to a destination&quot;&quot;&quot;</span>
     <span class="n">dest</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">tmpdir</span> <span class="o">/</span> <span class="s2">&quot;to_download.txt&quot;</span><span class="p">)</span>
@@ -1140,6 +1142,7 @@ <h4 id="url_2_text" class="doc_header"><code>url_2_text</code><a href="https://g
     <span class="k">assert</span> <span class="n">dest</span><span class="o">.</span><span class="n">read_text</span><span class="p">()</span> <span class="o">==</span> <span class="n">test_data_txt</span>
 
 
+<span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
 <span class="k">def</span> <span class="nf">test_download_empty</span><span class="p">(</span><span class="n">check_connection_github</span><span class="p">,</span> <span class="n">tmpdir</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;Test download of file without destination&quot;&quot;&quot;</span>
     <span class="n">dest</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">&quot;to_download.txt&quot;</span><span class="p">)</span>
@@ -1151,6 +1154,7 @@ <h4 id="url_2_text" class="doc_header"><code>url_2_text</code><a href="https://g
         <span class="n">dest</span><span class="o">.</span><span class="n">unlink</span><span class="p">()</span>
 
 
+<span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
 <span class="k">def</span> <span class="nf">test_url_2_text</span><span class="p">(</span><span class="n">check_connection_github</span><span class="p">):</span>
     <span class="sd">&quot;&quot;&quot;Test extraction of text from URL&quot;&quot;&quot;</span>
     <span class="k">assert</span> <span class="n">url_2_text</span><span class="p">(</span><span class="n">url_raw_txt</span><span class="p">)</span> <span class="o">==</span> <span class="n">test_data_txt</span>
@@ -1325,6 +1329,7 @@ <h4 id="read_csv_from_zip" class="doc_header"><code>read_csv_from_zip</code><a h
     <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span>
 
 
+<span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">github</span>
 <span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">parametrize</span><span class="p">(</span>
     <span class="s2">&quot;archive,csv&quot;</span><span class="p">,</span>
     <span class="p">[</span>

nbs/10_utils.ipynb

@@ -519,22 +519,7 @@
     },
     {
      "data": {
-      "application/javascript": [
-       "\n",
-       "    console.info(\"loading toggle event: 00d9724d-55e4-4ab8-b426-5e92f71f593c\")\n",
-       "    const toggle_hush = (e) =>{\n",
-       "        var op = document.querySelector(\".hush_output_00d9724d-55e4-4ab8-b426-5e92f71f593c\");\n",
-       "        if(op.style.display==\"none\"){\n",
-       "             op.style.display=\"block\"\n",
-       "\n",
-       "        } else {\n",
-       "            op.style.display=\"none\"\n",
-       "        }\n",
-       "\n",
-       "    }\n",
-       "    document.querySelector('.hush_toggle_00d9724d-55e4-4ab8-b426-5e92f71f593c').onclick=toggle_hush\n",
-       "    "
-      ],
+      "application/javascript": "\n    console.info(\"loading toggle event: 00d9724d-55e4-4ab8-b426-5e92f71f593c\")\n    const toggle_hush = (e) =>{\n        var op = document.querySelector(\".hush_output_00d9724d-55e4-4ab8-b426-5e92f71f593c\");\n        if(op.style.display==\"none\"){\n             op.style.display=\"block\"\n\n        } else {\n            op.style.display=\"none\"\n        }\n\n    }\n    document.querySelector('.hush_toggle_00d9724d-55e4-4ab8-b426-5e92f71f593c').onclick=toggle_hush\n    ",
       "text/plain": [
        "<IPython.core.display.Javascript object>"
       ]
@@ -589,22 +574,7 @@
     },
     {
      "data": {
-      "application/javascript": [
-       "\n",
-       "    console.info(\"loading toggle event: a5461483-be0b-4a73-8299-458f05a7c0d5\")\n",
-       "    const toggle_hush = (e) =>{\n",
-       "        var op = document.querySelector(\".hush_output_a5461483-be0b-4a73-8299-458f05a7c0d5\");\n",
-       "        if(op.style.display==\"none\"){\n",
-       "             op.style.display=\"block\"\n",
-       "\n",
-       "        } else {\n",
-       "            op.style.display=\"none\"\n",
-       "        }\n",
-       "\n",
-       "    }\n",
-       "    document.querySelector('.hush_toggle_a5461483-be0b-4a73-8299-458f05a7c0d5').onclick=toggle_hush\n",
-       "    "
-      ],
+      "application/javascript": "\n    console.info(\"loading toggle event: a5461483-be0b-4a73-8299-458f05a7c0d5\")\n    const toggle_hush = (e) =>{\n        var op = document.querySelector(\".hush_output_a5461483-be0b-4a73-8299-458f05a7c0d5\");\n        if(op.style.display==\"none\"){\n             op.style.display=\"block\"\n\n        } else {\n            op.style.display=\"none\"\n        }\n\n    }\n    document.querySelector('.hush_toggle_a5461483-be0b-4a73-8299-458f05a7c0d5').onclick=toggle_hush\n    ",
       "text/plain": [
        "<IPython.core.display.Javascript object>"
       ]
@@ -649,22 +619,7 @@
     },
     {
      "data": {
-      "application/javascript": [
-       "\n",
-       "    console.info(\"loading toggle event: 33d84ffd-6fef-48a7-8ddc-26ef5ec8513b\")\n",
-       "    const toggle_hush = (e) =>{\n",
-       "        var op = document.querySelector(\".hush_output_33d84ffd-6fef-48a7-8ddc-26ef5ec8513b\");\n",
-       "        if(op.style.display==\"none\"){\n",
-       "             op.style.display=\"block\"\n",
-       "\n",
-       "        } else {\n",
-       "            op.style.display=\"none\"\n",
-       "        }\n",
-       "\n",
-       "    }\n",
-       "    document.querySelector('.hush_toggle_33d84ffd-6fef-48a7-8ddc-26ef5ec8513b').onclick=toggle_hush\n",
-       "    "
-      ],
+      "application/javascript": "\n    console.info(\"loading toggle event: 33d84ffd-6fef-48a7-8ddc-26ef5ec8513b\")\n    const toggle_hush = (e) =>{\n        var op = document.querySelector(\".hush_output_33d84ffd-6fef-48a7-8ddc-26ef5ec8513b\");\n        if(op.style.display==\"none\"){\n             op.style.display=\"block\"\n\n        } else {\n            op.style.display=\"none\"\n        }\n\n    }\n    document.querySelector('.hush_toggle_33d84ffd-6fef-48a7-8ddc-26ef5ec8513b').onclick=toggle_hush\n    ",
       "text/plain": [
        "<IPython.core.display.Javascript object>"
       ]
@@ -1174,10 +1129,12 @@
    "outputs": [],
    "source": [
     "# exportest\n",
+    "@pytest.mark.github\n",
     "def test_get_url_size(check_connection_github):\n",
     "    assert get_url_size(url_raw_txt) == 264, f\"Wrong size for {url_raw_txt}\"\n",
     "\n",
     "\n",
+    "@pytest.mark.github\n",
     "def test_download_dest(check_connection_github, tmpdir):\n",
     "    \"\"\"Test download of file to a destination\"\"\"\n",
     "    dest = Path(tmpdir / \"to_download.txt\")\n",
@@ -1186,6 +1143,7 @@
     "    assert dest.read_text() == test_data_txt\n",
     "\n",
     "\n",
+    "@pytest.mark.github\n",
     "def test_download_empty(check_connection_github, tmpdir):\n",
     "    \"\"\"Test download of file without destination\"\"\"\n",
     "    dest = Path(\"to_download.txt\")\n",
@@ -1197,6 +1155,7 @@
     "        dest.unlink()\n",
     "\n",
     "\n",
+    "@pytest.mark.github\n",
     "def test_url_2_text(check_connection_github):\n",
     "    \"\"\"Test extraction of text from URL\"\"\"\n",
     "    assert url_2_text(url_raw_txt) == test_data_txt\n"
@@ -1807,6 +1766,7 @@
     "    assert len(df) == 100\n",
     "\n",
     "\n",
+    "@pytest.mark.github\n",
     "@pytest.mark.parametrize(\n",
     "    \"archive,csv\",\n",
     "    [\n",

nbs/41_nlp_data.ipynb

@@ -456,11 +456,13 @@
     "        with pytest.raises(FileExistsError):\n",
     "            textual = Textual.from_path(\"does_not_exist.txt\")\n",
     "\n",
+    "    @pytest.mark.github\n",
     "    def test_from_url(self, check_connection_github, local_textual):\n",
     "        \"\"\"Test extract Textual from URL\"\"\"\n",
     "        textual = Textual.from_url(GITHUB_TEST_TXT)\n",
     "        assert textual.text == local_textual.text, f\"URL text: {textual.text}\"\n",
     "\n",
+    "    @pytest.mark.github\n",
     "    def test_from_url_non_ascii(self, check_connection_github):\n",
     "        \"\"\"Test extract Textual from URL with non-ascii characters\"\"\"\n",
     "        textual = Textual.from_url(GITHUB_TEST_TXT_UTF8)\n",

nbs/42_nlp_pdf.ipynb

@@ -308,11 +308,13 @@
     "        extra_pages = set(pages_extracted) - set(str(p + 1) for p in pages)\n",
     "        assert extra_pages == set(), f\"Extra pages extracted: {extra_pages}\"\n",
     "\n",
+    "    @pytest.mark.github\n",
     "    def test_from_url(self, check_connection_github, local_textual):\n",
     "        \"\"\"Test extract Textual of PDF from URL\"\"\"\n",
     "        textual = TextualPDF.from_url(GITHUB_TEST_PDF)\n",
     "        assert textual.text == local_textual.text, f\"URL text: {textual.text}\"\n",
     "\n",
+    "    @pytest.mark.github\n",
     "    def test_from_url_pdf(self, check_connection_github, local_textual):\n",
     "        \"\"\"Test extract Textual of PDF from URL using from_url_pdf\"\"\"\n",
     "        textual = TextualPDF.from_url_pdf(GITHUB_TEST_PDF)\n",

pytest.ini

@@ -1,2 +1,6 @@
 [pytest]
-testpaths = test
+testpaths = test
+markers =
+    github: mark a test using data on github.
+    slow: mark test as slow.
+    magic: mark test using iPython magic.

test/nlp/test_data.py

@@ -52,11 +52,13 @@ def test_from_path_error(self):
         with pytest.raises(FileExistsError):
             textual = Textual.from_path("does_not_exist.txt")
 
+    @pytest.mark.github
     def test_from_url(self, check_connection_github, local_textual):
         """Test extract Textual from URL"""
         textual = Textual.from_url(GITHUB_TEST_TXT)
         assert textual.text == local_textual.text, f"URL text: {textual.text}"
 
+    @pytest.mark.github
     def test_from_url_non_ascii(self, check_connection_github):
         """Test extract Textual from URL with non-ascii characters"""
         textual = Textual.from_url(GITHUB_TEST_TXT_UTF8)

test/nlp/test_pdf.py

@@ -79,11 +79,13 @@ def test_from_path_pdf_pages(self, pages):
         extra_pages = set(pages_extracted) - set(str(p + 1) for p in pages)
         assert extra_pages == set(), f"Extra pages extracted: {extra_pages}"
 
+    @pytest.mark.github
     def test_from_url(self, check_connection_github, local_textual):
         """Test extract Textual of PDF from URL"""
         textual = TextualPDF.from_url(GITHUB_TEST_PDF)
         assert textual.text == local_textual.text, f"URL text: {textual.text}"
 
+    @pytest.mark.github
     def test_from_url_pdf(self, check_connection_github, local_textual):
         """Test extract Textual of PDF from URL using from_url_pdf"""
         textual = TextualPDF.from_url_pdf(GITHUB_TEST_PDF)