Merge pull request #736 from flairNLP/add-iceland

Add `ISL`

Author: addie9800

docs/supported_publishers.md

@@ -1549,6 +1549,44 @@
 </table>
 
 
+## ISL-Publishers
+
+<table class="publishers isl">
+  <thead>
+    <tr>
+      <th>Class&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</th>
+      <th>Name&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</th>
+      <th>URL&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;</th>
+      <th>Languages</th>
+      <th>Missing&#160;Attributes</th>
+      <th>Additional&#160;Attributes&#160;&#160;&#160;&#160;</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>
+        <code>Morgunbladid</code>
+      </td>
+      <td>
+        <div>Morgunbladid</div>
+      </td>
+      <td>
+        <a href="https://www.mbl.is/">
+          <span>www.mbl.is</span>
+        </a>
+      </td>
+      <td>
+        <code>is</code>
+      </td>
+      <td>
+        <code>topics</code>
+      </td>
+      <td>&#160;</td>
+    </tr>
+  </tbody>
+</table>
+
+
 ## IT-Publishers
 
 <table class="publishers it">

src/fundus/parser/utility.py

@@ -642,7 +642,12 @@ def get_versions_from_node(
 
 def parse_versions(img_node: lxml.html.HtmlElement, size_pattern: Optional[Pattern[str]] = None) -> List[ImageVersion]:
     # parse img
-    if (default_width := img_node.get("width")) and (default_height := img_node.get("height")):
+    if (
+        (default_width := img_node.get("width"))
+        and not default_width == "auto"
+        and (default_height := img_node.get("height"))
+        and not default_height == "auto"
+    ):
         ratio = float(default_width) / float(default_height)
     else:
         ratio = None

src/fundus/publishers/__init__.py

@@ -15,6 +15,7 @@
 from fundus.publishers.fr import FR
 from fundus.publishers.gl import GL
 from fundus.publishers.ind import IND
+from fundus.publishers.isl import ISL
 from fundus.publishers.it import IT
 from fundus.publishers.jp import JP
 from fundus.publishers.lt import LT
@@ -68,6 +69,7 @@ class PublisherCollection(metaclass=PublisherCollectionMeta):
     us = US
     uk = UK
     fr = FR
+    isl = ISL
     gl = GL
     ch = CH
     lt = LT

src/fundus/publishers/isl/__init__.py

@@ -0,0 +1,24 @@
+from fundus.publishers.base_objects import Publisher, PublisherGroup
+from fundus.scraping.url import RSSFeed, Sitemap
+
+from .morgunbladid import MorgunbladidParser
+
+# noinspection PyPep8Naming
+
+
+class ISL(metaclass=PublisherGroup):
+    default_language = "is"
+
+    Morgunbladid = Publisher(
+        name="Morgunbladid",
+        domain="https://www.mbl.is/",
+        parser=MorgunbladidParser,
+        sources=[
+            Sitemap("https://www.mbl.is/mm/sitemap.xml"),
+            RSSFeed("https://www.mbl.is/feeds/fp/"),
+            RSSFeed("https://www.mbl.is/feeds/innlent/"),
+            RSSFeed("https://www.mbl.is/feeds/erlent/"),
+            RSSFeed("https://www.mbl.is/feeds/english/"),
+            RSSFeed("https://www.mbl.is/feeds/helst/"),
+        ],
+    )

src/fundus/publishers/isl/morgunbladid.py

@@ -0,0 +1,54 @@
+import datetime
+import re
+from typing import List, Optional
+
+from lxml.cssselect import CSSSelector
+from lxml.etree import XPath
+
+from fundus.parser import ArticleBody, BaseParser, Image, ParserProxy, attribute
+from fundus.parser.utility import (
+    extract_article_body_with_selector,
+    generic_author_parsing,
+    generic_date_parsing,
+    generic_topic_parsing,
+    image_extraction,
+)
+
+
+class MorgunbladidParser(ParserProxy):
+    class V1(BaseParser):
+        _summary_selector = XPath("//div[@class='main-layout']//div[@class='is-merking']/p")
+        _paragraph_selector = XPath(
+            "//div[@class='main-layout' or @data-element-type='body-facts']" "/p[not(a and not(text()))]"
+        )
+        _subheadline_selector = XPath("//div[@class='main-layout' or @class='et_pb_text_inner']/h3")
+
+        @attribute
+        def body(self) -> Optional[ArticleBody]:
+            return extract_article_body_with_selector(
+                self.precomputed.doc,
+                subheadline_selector=self._subheadline_selector,
+                paragraph_selector=self._paragraph_selector,
+            )
+
+        @attribute
+        def authors(self) -> List[str]:
+            return generic_author_parsing(self.precomputed.ld.bf_search("author"))
+
+        @attribute
+        def publishing_date(self) -> Optional[datetime.datetime]:
+            return generic_date_parsing(self.precomputed.ld.bf_search("datePublished"))
+
+        @attribute
+        def title(self) -> Optional[str]:
+            return self.precomputed.ld.bf_search("headline")
+
+        @attribute
+        def images(self) -> List[Image]:
+            return image_extraction(
+                doc=self.precomputed.doc,
+                paragraph_selector=self._paragraph_selector,
+                image_selector=XPath("//div[@class='image']//img"),
+                caption_selector=XPath("./ancestor::div[contains(@class, 'newsitem-image')]//span[@class='caption']"),
+                author_selector=XPath("./ancestor::div[contains(@class, 'newsitem-image')]//span[@class='credit']"),
+            )