Pre-Release: add precommit-hook configuration for linters, doc styles

.flake8

@@ -0,0 +1,36 @@
+[flake8]
+ignore =
+    # Refers to the max-line length. Let's suppress the error and simply
+    # let black take care on how it wants to format the lines.
+    E501,
+
+    # Refers to "line break before binary operator".
+    # Similar to above, let black take care of the formatting.
+    W503,
+
+    # Refers to "necessary dict call - rewrite as a literal".
+    C408,
+
+    # To be addressed:
+    D100,  # Missing docstring in public module
+    D101,  # Missing docstring in public class
+    D102,  # Missing docstring in public method
+    D103,  # Missing docstring in public function
+    D104,  # Missing docstring in public package
+    D105,  # Missing docstring in magic method
+    D107,  # Missing docstring in __init__
+    D200,  # One-line docstring should fit on one line with quotes
+    D202,  # No blank lines allowed after function docstring
+    D205,  # 1 blank line required between summary line and description
+    D209,  # Multi-line docstring closing quotes should be on a separate line
+    D400,  # First line should end with a period
+    D401,  # First line should be in imperative mood
+    D402   # First line should not be the function's "signature"
+
+per-file-ignores =
+    # F401: Ignore "imported but unused" errors in __init__ files, as those
+    # imports are there to expose submodule functions so they can be imported
+    # directly from that module
+    # F403: Ignore * imports in these files
+    scrapy_poet/__init__.py:F401,F403
+    scrapy_poet/page_inputs/__init__.py:F401,F403

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# Contains commits to be ignored due to linting
+
+# https://github.com/scrapinghub/scrapy-poet/pull/68
+58c903617911b3209ad68bfefe3fa1a86be629f4
+7249a133722d1115111a8bbb3b02080a892483f2

.github/workflows/test.yml

@@ -57,7 +57,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ['3.10']
-        tox-job: ["mypy", "docs"]
+        tox-job: ["mypy", "docs", "linters"]
 
     steps:
     - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - hooks:
+      - id: black
+        language_version: python3
+    repo: https://github.com/ambv/black
+    rev: 22.3.0
+  - hooks:
+      - id: isort
+        language_version: python3
+    repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+  - hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-debugger
+          - flake8-docstrings
+          - flake8-string-format
+    repo: https://github.com/pycqa/flake8
+    rev: 4.0.1

README.rst

@@ -60,3 +60,21 @@ Add the following inside Scrapy's ``settings.py`` file:
     DOWNLOADER_MIDDLEWARES = {
         "scrapy_poet.InjectionMiddleware": 543,
     }
+
+Developing
+==========
+
+Setup your local Python environment via:
+
+1. `pip install -r requirements-dev.txt`
+2. `pre-commit install`
+
+Now everytime you perform a `git commit`, these tools will run against the
+staged files:
+
+* `black`
+* `isort`
+* `flake8`
+
+You can also directly invoke `pre-commit run --all-files` or `tox -e linters`
+to run them without performing a commit.

docs/conf.py

@@ -12,19 +12,20 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../'))
+
+sys.path.insert(0, os.path.abspath("../"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = u'scrapy-poet'
-copyright = u'2022, Zyte'
-author = u'Zyte'
+project = "scrapy-poet"
+copyright = "2022, Zyte"
+author = "Zyte"
 
 # The short X.Y version
-version = u''
+version = ""
 # The full version, including alpha/beta/rc tags
-release = u'0.3.0'
+release = "0.3.0"
 
 
 # -- General configuration ---------------------------------------------------
@@ -37,24 +38,24 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.githubpages',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.githubpages",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -66,7 +67,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
@@ -77,12 +78,13 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom themes here, relative to this directory.
 # Add path to the RTD explicitly to robustify builds (otherwise might
 # fail in a clean Debian build env)
-import sphinx_rtd_theme
+import sphinx_rtd_theme  # noqa: E402
+
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -110,7 +112,7 @@
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'scrapy-poetdoc'
+htmlhelp_basename = "scrapy-poetdoc"
 
 
 # -- Options for LaTeX output ------------------------------------------------
@@ -119,15 +121,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -137,19 +136,21 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'scrapy-poet.tex', u'scrapy-poet Documentation',
-     u'Scrapinghub', 'manual'),
+    (
+        master_doc,
+        "scrapy-poet.tex",
+        "scrapy-poet Documentation",
+        "Scrapinghub",
+        "manual",
+    ),
 ]
 
 
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'scrapy-poet', u'scrapy-poet Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "scrapy-poet", "scrapy-poet Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output ----------------------------------------------
@@ -158,9 +159,15 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'scrapy-poet', u'scrapy-poet Documentation',
-     author, 'scrapy-poet', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "scrapy-poet",
+        "scrapy-poet Documentation",
+        author,
+        "scrapy-poet",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 
@@ -179,21 +186,27 @@
 # epub_uid = ''
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for intersphinx extension ---------------------------------------
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3', None, ),
-    'scrapy': ('https://docs.scrapy.org/en/latest', None, ),
-    'web-poet': ('https://web-poet.readthedocs.io/en/latest/', None),
-    'url-matcher': ('https://url-matcher.readthedocs.io/en/stable/', None),
+    "python": (
+        "https://docs.python.org/3",
+        None,
+    ),
+    "scrapy": (
+        "https://docs.scrapy.org/en/latest",
+        None,
+    ),
+    "web-poet": ("https://web-poet.readthedocs.io/en/latest/", None),
+    "url-matcher": ("https://url-matcher.readthedocs.io/en/stable/", None),
 }
 
 autodoc_default_options = {
-    'special-members': '__init__,__call__',
+    "special-members": "__init__,__call__",
     # 'undoc-members': True,
-    'exclude-members': '__weakref__'
+    "exclude-members": "__weakref__",
 }

docs/index.rst

@@ -25,9 +25,9 @@ To get started, see :ref:`intro-install` and :ref:`intro-tutorial`.
 
 :ref:`license` is BSD 3-clause.
 
-.. _`AutoExtract`: https://scrapinghub.com/autoextract
-.. _`Splash`: https://scrapinghub.com/splash
-.. _`web-poet`: https://github.com/scrapinghub/web-poet
+.. _AutoExtract: https://scrapinghub.com/autoextract
+.. _Splash: https://scrapinghub.com/splash
+.. _web-poet: https://github.com/scrapinghub/web-poet
 .. _docs: https://web-poet.readthedocs.io/en/stable/
 
 .. toctree::

docs/intro/advanced-tutorial.rst

@@ -1,4 +1,4 @@
-.. _`intro-advanced-tutorial`:
+.. _intro-advanced-tutorial:
 
 =================
 Advanced Tutorial
@@ -15,7 +15,7 @@ These are mainly achieved by **scrapy-poet** implementing **providers** for them
     * :class:`scrapy_poet.page_input_providers.HttpClientProvider`
     * :class:`scrapy_poet.page_input_providers.PageParamsProvider`
 
-.. _`intro-additional-requests`:
+.. _intro-additional-requests:
 
 Additional Requests
 ===================

docs/intro/basic-tutorial.rst

@@ -1,4 +1,4 @@
-.. _`intro-basic-tutorial`:
+.. _intro-basic-tutorial:
 
 ==============
 Basic Tutorial

docs/intro/install.rst

@@ -1,4 +1,4 @@
-.. _`intro-install`:
+.. _intro-install:
 
 ============
 Installation

docs/license.rst

@@ -1,4 +1,4 @@
-.. _`license`:
+.. _license:
 
 =======
 License

docs/overrides.rst

@@ -1,4 +1,4 @@
-.. _`overrides`:
+.. _overrides:
 
 =========
 Overrides

docs/providers.rst

@@ -1,4 +1,4 @@
-.. _`providers`:
+.. _providers:
 
 =========
 Providers

docs/settings.rst

@@ -1,4 +1,4 @@
-.. _`settings`:
+.. _settings:
 
 Settings
 ========