Make cchardet optional (#211)

Cchardet is not compatible with Python 3.11+ (there is an alpha release that works on 3.11 and 3.12, but it is now old too), and we need to make and keep this package easier to install, update, and secure. Cchardet seemed very dependable for a long time, but its maintainer eventually took a long break. It's pretty frozen, and we cannot depend on it as a hard requirement.

The detection test had to be amended to a different encoding in order to get something that is supported and reliably detected across all detector options.

Partially covers #196.

Author: Mr0grog

.circleci/config.yml

@@ -15,9 +15,13 @@ commands:
         description: Install docs dependencies
         type: boolean
         default: false
+      chardet:
+        type: string
+        default: ""
     steps:
       - restore_cache:
           keys:
+            - cache-v4-{{ arch }}-python<< parameters.python-version >>-<< parameters.chardet >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-{{ checksum "requirements-docs.txt" }}
             - cache-v4-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-{{ checksum "requirements-docs.txt" }}
             - cache-v4-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-
             - cache-v4-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-
@@ -35,6 +39,14 @@ commands:
             pip install .[server,dev] --no-binary lxml
             pip install -r requirements-experimental.txt
 
+      - when:
+          condition: << parameters.chardet >>
+          steps:
+            - run:
+                command: |
+                  . ~/venv/bin/activate
+                  pip install << parameters.chardet >>
+
       # Docs dependencies are only compatible on Python 3.10+, so only install
       # them on demand.
       - when:
@@ -46,7 +58,7 @@ commands:
                   pip install -r requirements-docs.txt
 
       - save_cache:
-          key: cache-v4-{{ arch }}-python<< parameters.python-version >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-{{ checksum "requirements-docs.txt" }}
+          key: cache-v4-{{ arch }}-python<< parameters.python-version >>-<< parameters.chardet >>-{{ checksum "requirements.txt" }}-{{ checksum "requirements-server.txt" }}-{{ checksum "requirements-dev.txt" }}-{{ checksum "requirements-experimental.txt" }}-{{ checksum "requirements-docs.txt" }}
           paths:
             - ~/venv
 
@@ -55,13 +67,17 @@ jobs:
     parameters:
       python-version:
         type: string
+      chardet:
+        type: string
+        default: ""
     working_directory: ~/web-monitoring-diff
     docker:
       - image: cimg/python:<< parameters.python-version >>
     steps:
       - checkout
       - setup_pip:
           python-version: << parameters.python-version >>
+          chardet: << parameters.chardet >>
       - run:
           name: Tests
           command: |
@@ -179,7 +195,23 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              python-version: ["3.7", "3.8", "3.9", "3.10"]
+              python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+              chardet: ["", "cchardet", "faust-cchardet"]
+            exclude:
+              - python-version: "3.7"
+                chardet: "faust-cchardet"
+              - python-version: "3.8"
+                chardet: "faust-cchardet"
+              - python-version: "3.9"
+                chardet: "faust-cchardet"
+              - python-version: "3.11"
+                chardet: "cchardet"
+              # - python-version: "3.12"
+              #   chardet: "cchardet"
+              # - python-version: "3.13"
+              #   chardet: "cchardet"
+              # - python-version: "3.14"
+              #   chardet: "cchardet"
       - lint:
           filters:
             tags:

Dockerfile

@@ -6,7 +6,7 @@
 # We separate them out so that the final `release` image can layer on top of
 # this one without needing compiler-related packages.
 ##
-FROM python:3.10.11-slim as base
+FROM python:3.10.17-slim as base
 LABEL maintainer="enviroDGI@gmail.com"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 WORKDIR /app
 
 RUN pip install --upgrade pip
+RUN pip install cchardet
 # Copy the requirements.txt alone into the container at /app
 # so that they can be cached more aggressively than the rest of the source.
 ADD requirements.txt /app

README.md

@@ -72,6 +72,15 @@ This package was originally built as a component of EDGI’s [Web Monitoring Pro
     pip install -r requirements-experimental.txt
     ```
 
+4. (Optional) If you are using the diff *server* and want high-performance character encoding detection, install [`cchardet`](https://pypi.org/project/cchardet). Note that it only supports Python 3.10 and *earlier* at the time of this writing. An alpha release supports up to Python 3.12.
+
+    ```sh
+    pip install cchardet
+
+    # Or, for Python 3.11 or 3.12:
+    pip install cchardet==2.2.0a2
+    ```
+
 
 ## Basic Usage
 

requirements-server.txt

@@ -7,7 +7,7 @@
 #
 # It only exists to keep the list of dependencies in a separate file from
 # setup.py.
-cchardet >=2.1.6,<3
+chardet >=5.0.0,<6
 pycurl >=7.43,<8
 sentry-sdk >=1.0.0,<2.0
 tornado >=6.0.0,<7

web_monitoring_diff/server/server.py

@@ -10,7 +10,6 @@
 import os
 import pycurl
 import re
-import cchardet
 import sentry_sdk
 import signal
 import sys
@@ -25,6 +24,17 @@
 from ..exceptions import UndiffableContentError, UndecodableContentError
 from ..utils import shutdown_executor_in_loop, Signal
 
+# Where possible, use cchardet (or faust-cchardet) for performance.
+# Unfortunately these aren't supported in the latest Python vesions, so we also
+# fall back to something in pure Python for those cases. There are a few
+# options there, but `chardet` offers the best accuracy/performance tradeoff
+# when operating on truncated/small content (we always truncate).
+# (Performance measurements as of mid-2024.)
+try:
+    import cchardet as chardet
+except ImportError:
+    import chardet
+
 logger = logging.getLogger(__name__)
 
 # Track errors with Sentry.io. It will automatically detect the `SENTRY_DSN`
@@ -641,11 +651,11 @@ def _extract_encoding(headers, content):
     if encoding:
         encoding = encoding.strip()
     if not encoding and content:
-        # try to identify encoding using cchardet. Use up to 18kb of the
+        # try to identify encoding using chardet. Use up to 18kb of the
         # content for detection. Its not necessary to use the full content
         # as it could be huge. Also, if you use too little, detection is not
         # accurate.
-        detected = cchardet.detect(content[:18432])
+        detected = chardet.detect(content[:18432])
         if detected:
             detected_encoding = detected.get('encoding')
             if detected_encoding:

web_monitoring_diff/tests/test_server_exc_handling.py

@@ -342,10 +342,10 @@ def test_extract_encoding_from_body(self):
         # Polish content without any content-type headers or meta tag.
         headers = {}
         body = """<html><head><title>TITLE</title></head>
-        <i>czyli co zrobić aby zobaczyć w tekstach polskie litery.</i>
-        Obowiązku czytania nie ma, ale wiele może wyjaśnić.
-        <body></body>""".encode('iso-8859-2')
-        assert df._extract_encoding(headers, body) == 'iso-8859-2'
+         <i>И така, какво да направите, за да видите българските букви в текстовете?</i>
+         Четенето не е задължително, но може да обясни много.
+         <body></body>""".encode('iso-8859-5')
+        assert df._extract_encoding(headers, body) == 'iso-8859-5'
 
     def test_diff_content_with_null_bytes(self):
         response = self.fetch('/html_source_dmp?format=json&'