Merge remote-tracking branches 'benoitc/pr/3217', 'benoitc/pr/3110', …

…'benoitc/pr/3157', 'benoitc/pr/3192', 'benoitc/pr/3210', 'benoitc/pr/3253', 'benoitc/pr/3254', 'benoitc/pr/3255', 'benoitc/pr/3257' and 'benoitc/pr/3258' into integration-v22.1.0

.github/workflows/tox.yml

@@ -19,7 +19,7 @@ jobs:
          - ubuntu-latest
          # not defaulting to macos-latest: Python <= 3.9 was missing from macos-14 @ arm64
          - macos-13
-         # Not testing Windows, because tests need Unix-only fcntl, grp, pwd, etc.
+         # Not testing Windows, because tests need Unix-only non-blocking pipes, grp, pwd, etc.
         python-version:
          # CPython <= 3.7 is EoL since 2023-06-27
          - "3.7"

.gitignore

@@ -7,6 +7,7 @@
 .tox
 __pycache__
 build
+docs/_build
 coverage.xml
 dist
 examples/frameworks/django/testing/testdb.sql

.pylintrc

@@ -15,9 +15,8 @@ disable=
     bad-mcs-classmethod-argument,
     bare-except,
     broad-except,
-    duplicate-bases,
     duplicate-code,
-    eval-used,
+    superfluous-parens,
     fixme,
     import-error,
     import-outside-toplevel,
@@ -47,9 +46,7 @@ disable=
     ungrouped-imports,
     unused-argument,
     useless-object-inheritance,
-    useless-import-alias,
-    comparison-with-callable,
     try-except-raise,
     consider-using-with,
     consider-using-f-string,
-    unspecified-encoding
+    unspecified-encoding,

MANIFEST.in

@@ -10,4 +10,6 @@ recursive-include examples *
 recursive-include docs *
 recursive-include examples/frameworks *
 recursive-exclude * __pycache__
+recursive-exclude docs/build *
+recursive-exclude docs/_build *
 recursive-exclude * *.py[co]

appveyor.yml

@@ -13,7 +13,7 @@ environment:
     #- TOXENV: run-entrypoint
     #  PYTHON: "C:\\Python38-x64"
     # Windows is not ready for testing!!!
-    # Python's fcntl, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only.
+    # Python's non-blocking pipes, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only.
     #- TOXENV: py35
     #  PYTHON: "C:\\Python35-x64"
     #- TOXENV: py36

docs/Makefile

@@ -1,5 +1,7 @@
 # Makefile for Sphinx documentation
 #
+# if you want to compare this file to current sphinx defaults, recreate it:
+# BUILDDIR=build sphinx-quickstart --sep --extensions=gunicorn_ext --templatedir=_templates --makefile --batchfile --no-use-make-mode --master=index
 
 # You can set these variables from the command line.
 PYTHON        = python

docs/README.rst

@@ -6,8 +6,8 @@ Requirements
 
 To generate documentation you need to install:
 
- - Python >= 3.4
- - Sphinx (http://sphinx-doc.org/)
+ - Python >= 3.7
+ - Sphinx (https://www.sphinx-doc.org/)
 
 
 Generate html

docs/source/deploy.rst

@@ -246,20 +246,24 @@ to the newly created unix socket:
     After=network.target
 
     [Service]
+    # gunicorn can let systemd know when it is ready
     Type=notify
+    NotifyAccess=main
     # the specific user that our service will run as
     User=someuser
     Group=someuser
-    # another option for an even more restricted service is
-    # DynamicUser=yes
-    # see http://0pointer.net/blog/dynamic-users-with-systemd.html
+    # this user can be transiently created by systemd
+    # DynamicUser=true
     RuntimeDirectory=gunicorn
-    WorkingDirectory=/home/someuser/applicationroot
-    ExecStart=/usr/bin/gunicorn applicationname.wsgi
+    WorkingDirectory=~
+    # using the @ prefix, you can set venv path via argv[0]
+    ExecStart=@/usr/bin/python3 ${HOME}/bin/python3 -m gunicorn --chdir appdir applicationname.wsgi
     ExecReload=/bin/kill -s HUP $MAINPID
     KillMode=mixed
     TimeoutStopSec=5
     PrivateTmp=true
+    # if your app does not need administrative capabilities, let systemd know
+    # ProtectSystem=strict
 
     [Install]
     WantedBy=multi-user.target
@@ -272,11 +276,12 @@ to the newly created unix socket:
     [Socket]
     ListenStream=/run/gunicorn.sock
     # Our service won't need permissions for the socket, since it
-    # inherits the file descriptor by socket activation
-    # only the nginx daemon will need access to the socket
+    # inherits the file descriptor by socket activation.
+    # Only the nginx daemon will need access to the socket:
     SocketUser=www-data
-    # Optionally restrict the socket permissions even more.
-    # SocketMode=600
+    SocketGroup=www-data
+    # Once the user/group is correct, restrict the permissions:
+    SocketMode=0660
 
     [Install]
     WantedBy=sockets.target

docs/source/faq.rst

@@ -11,7 +11,9 @@ How do I set SCRIPT_NAME?
 -------------------------
 
 By default ``SCRIPT_NAME`` is an empty string. The value could be set by
-setting ``SCRIPT_NAME`` in the environment or as an HTTP header.
+setting ``SCRIPT_NAME`` in the environment or as an HTTP header. Note that
+this headers contains and underscore, so it is only accepted from trusted
+forwarders listed in the ``forwarded-allow-ips`` setting.
 
 
 Server Stuff

docs/source/install.rst

@@ -96,68 +96,42 @@ advantages:
   rolled back in case of incompatibility. The package can also be purged
   entirely from the system in seconds.
 
-stable ("buster")
-------------------
+stable (as of 2024, "bookworm")
+-------------------------------
 
-The version of Gunicorn in the Debian_ "stable" distribution is 19.9.0
-(December 2020). You can install it using::
+The version of Gunicorn in the Debian_ "stable" distribution is 20.1.0
+(2021-04-28). You can install it using::
 
-    $ sudo apt-get install gunicorn3
-
-You can also use the most recent version 20.0.4 (December 2020) by using
-`Debian Backports`_. First, copy the following line to your
-``/etc/apt/sources.list``::
-
-    deb http://ftp.debian.org/debian buster-backports main
-
-Then, update your local package lists::
-
-    $ sudo apt-get update
-
-You can then install the latest version using::
-
-    $ sudo apt-get -t buster-backports install gunicorn
-
-oldstable ("stretch")
----------------------
-
-While Debian releases newer than Stretch will give you gunicorn with Python 3
-support no matter if you install the gunicorn or gunicorn3 package for Stretch
-you specifically have to install gunicorn3 to get Python 3 support.
-
-The version of Gunicorn in the Debian_ "oldstable" distribution is 19.6.0
-(December 2020). You can install it using::
-
-    $ sudo apt-get install gunicorn3
+    $ sudo apt-get install gunicorn
 
-You can also use the most recent version 19.7.1 (December 2020) by using
+You may have access to a more recent packaged version by using
 `Debian Backports`_. First, copy the following line to your
 ``/etc/apt/sources.list``::
 
-    deb http://ftp.debian.org/debian stretch-backports main
+    deb http://ftp.debian.org/debian bookworm-backports main
 
 Then, update your local package lists::
 
     $ sudo apt-get update
 
-You can then install the latest version using::
+You can then install the latest available version using::
 
-    $ sudo apt-get -t stretch-backports install gunicorn3
+    $ sudo apt-get -t bookworm-backports install gunicorn
 
-Testing ("bullseye") / Unstable ("sid")
----------------------------------------
+Testing (as of 2024, "trixie") / Unstable ("sid")
+-------------------------------------------------
 
-"bullseye" and "sid" contain the latest released version of Gunicorn 20.0.4
-(December 2020). You can install it in the usual way::
+"trixie" and "sid" ship the most recently packaged version of Gunicorn 20.1.0
+(2021-04-28). You can install it in the usual way::
 
     $ sudo apt-get install gunicorn
 
 
 Ubuntu
 ======
 
-Ubuntu_ 20.04 LTS (Focal Fossa) or later contains the Gunicorn package by
-default 20.0.4 (December 2020) so that you can install it in the usual way::
+Ubuntu_ 20.04 LTS (Focal Fossa) and later ship packages similar to Debian
+so that you can install it in the usual way::
 
     $ sudo apt-get update
     $ sudo apt-get install gunicorn

docs/source/news.rst

@@ -2,6 +2,16 @@
 Changelog
 =========
 
+22.0.1 - 2TBDTBDTBD
+===================
+
+- the SCRIPT_NAME header when received from allowed forwarders is no longer restricted for containing an underscore
+
+*** NOTE ***
+
+- This mitigates a regression that appeared first in the 22.0.0 release
+- Review your ``forwarded-allow-ips`` setting if you are still not seeing the SCRIPT_NAME transmitted
+
 22.0.0 - 2024-04-17
 ===================
 

docs/source/settings.rst

@@ -1232,17 +1232,18 @@ the headers defined here can not be passed directly from the client.
 
 **Command line:** ``--forwarded-allow-ips STRING``
 
-**Default:** ``'127.0.0.1'``
+**Default:** ``'127.0.0.1,::1'``
 
 Front-end's IPs from which allowed to handle set secure headers.
 (comma separate).
 
-Set to ``*`` to disable checking of Front-end IPs (useful for setups
+Set to ``*`` to disable checking of Front-end IPs. This is useful for setups
 where you don't know in advance the IP address of Front-end, but
-you still trust the environment).
+instead have ensured via other means that none other than your
+authorized Front-ends can access gunicorn.
 
 By default, the value of the ``FORWARDED_ALLOW_IPS`` environment
-variable. If it is not defined, the default is ``"127.0.0.1"``.
+variable. If it is not defined, the default is ``"127.0.0.1,::1"``.
 
 .. note::
 
@@ -1396,6 +1397,23 @@ The variables are passed to the PasteDeploy entrypoint. Example::
 
 .. versionadded:: 19.7
 
+.. _refuse-obsolete-folding:
+
+``refuse_obsolete_folding``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Command line:** ``--refuse-obsolete-folding``
+
+**Default:** ``False``
+
+Refuse requests employing obsolete HTTP line folding mechanism
+
+The mechanism was deprecated by rfc7230 Section 3.2.4.
+
+Safe to enable if you only ever want to serve standards compliant HTTP clients.
+
+.. versionadded:: 22.1.0
+
 .. _strip-header-spaces:
 
 ``strip_header_spaces``
@@ -1410,7 +1428,7 @@ Strip spaces present between the header name and the the ``:``.
 This is known to induce vulnerabilities and is not compliant with the HTTP/1.1 standard.
 See https://portswigger.net/research/http-desync-attacks-request-smuggling-reborn.
 
-Use with care and only if necessary. May be removed in a future version.
+Use with care and only if necessary. Deprecated; scheduled for removal in 25.0.0
 
 .. versionadded:: 20.0.1
 
@@ -1429,9 +1447,13 @@ This permits request methods of length less than 3 or more than 20,
 methods with lowercase characters or methods containing the # character.
 HTTP methods are case sensitive by definition, and merely uppercase by convention.
 
-This option is provided to diagnose backwards-incompatible changes.
+If unset, Gunicorn will apply nonstandard restrictions and cause 400 response status
+in cases where otherwise 501 status is expected. While this option does modify that
+behaviour, it should not be depended upon to guarantee standards-compliant behaviour.
+Rather, it is provided temporarily, to assist in diagnosing backwards-incompatible
+changes around the incomplete application of those restrictions.
 
-Use with care and only if necessary. May be removed in a future version.
+Use with care and only if necessary. Temporary; scheduled for removal in 24.0.0
 
 .. versionadded:: 22.0.0
 
@@ -1450,7 +1472,8 @@ This disables the refusal of likely malformed request lines.
 It is unusual to specify HTTP 1 versions other than 1.0 and 1.1.
 
 This option is provided to diagnose backwards-incompatible changes.
-Use with care and only if necessary. May be removed in a future version.
+Use with care and only if necessary. Temporary; the precise effect of this option may
+change in a future version, or it may be removed altogether.
 
 .. versionadded:: 22.0.0
 
@@ -1469,10 +1492,29 @@ HTTP methods are case sensitive by definition, and merely uppercase by conventio
 
 This option is provided because previous versions of gunicorn defaulted to this behaviour.
 
-Use with care and only if necessary. May be removed in a future version.
+Use with care and only if necessary. Deprecated; scheduled for removal in 24.0.0
 
 .. versionadded:: 22.0.0
 
+.. _forwarder-headers:
+
+``forwarder_headers``
+~~~~~~~~~~~~~~~~~~~~~
+
+**Command line:** ``--forwarder-headers``
+
+**Default:** ``'SCRIPT_NAME'``
+
+A list containing upper-case header field names that the front-end proxy
+sets, to be used in WSGI environment.
+
+If headers named in this list are not present in the request, they will be ignored.
+
+This option can be used to transfer SCRIPT_NAME and REMOTE_USER.
+
+It is important that your front-end proxy configuration ensures that
+the headers defined here can not be passed directly from the client.
+
 .. _header-map:
 
 ``header_map``
@@ -1490,9 +1532,13 @@ the same environment variable will dangerously confuse applications as to which
 
 The safe default ``drop`` is to silently drop headers that cannot be unambiguously mapped.
 The value ``refuse`` will return an error if a request contains *any* such header.
-The value ``dangerous`` matches the previous, not advisabble, behaviour of mapping different
+The value ``dangerous`` matches the previous, not advisable, behaviour of mapping different
 header field names into the same environ name.
 
+If the source IP is permitted by ``forwarded-allow-ips``, *and* the header name is
+present in ``forwarder-headers``, the header is mapped into environment regardless of
+the state of this setting.
+
 Use with care and only if necessary and after considering if your problem could
 instead be solved by specifically renaming or rewriting only the intended headers
 on a proxy in front of Gunicorn.
@@ -1512,9 +1558,18 @@ Process requests with both Transfer-Encoding and Content-Length
 
 This is known to induce vulnerabilities, but not strictly forbidden by RFC9112.
 
-Use with care and only if necessary. May be removed in a future version.
+In any case, the connection is closed after the malformed request,
+as it is unclear if and at which boundary additional requests start.
+
+Use with care and only if necessary.
+Temporary; will be changed or removed in a future version.
 
 .. versionadded:: 22.0.0
+.. versionchanged: 22.1.0
+   The newly added rejection of invalid and dangerous characters CR, LF and NUL in
+   header field values is also controlled with this setting. rfc9110 permits both
+   rejecting and SP-replacing. With this option set, Gunicorn passes the field value
+   unchanged. With this option unset, Gunicorn rejects the request.
 
 Server Socket
 -------------

gunicorn/app/wsgiapp.py

@@ -12,6 +12,8 @@
 
 class WSGIApplication(Application):
     def init(self, parser, opts, args):
+        # pylint: disable=cyclic-import
+
         self.app_uri = None
 
         if opts.paste:
@@ -48,6 +50,8 @@ def load_wsgiapp(self):
         return util.import_app(self.app_uri)
 
     def load_pasteapp(self):
+        # pylint: disable=cyclic-import
+
         from .pasterapp import get_wsgi_app
         return get_wsgi_app(self.app_uri, defaults=self.cfg.paste_global_conf)