ci(check-fixtures): parallelise fixture checking commands

Introduces several independent `run-tests.sh` fixture-checking commands
in order to speed up fixture checking by parallelisation.

Renames `run-tests.sh` script options and CI rules to better separate
data checks, formatting checks and linting checks.

Adds data formatting checks and fixes several JSON data files.

Adds `shfmt` formatting checks, `commitlint`, flake8` and `yamllint`
linting checks.

Removes `pydocstyle` formating checks since we moved to `black` code
formatter.

Introduces `/run-tests.sh --help` explaining all the checking options.

Updates CI environment to Ubuntu 24.04 and latest actions
(`actions/checkout@v4`, `actions/setup-node@v4`,
`actions/setup-python@v5`).

Amends `.editorconfig` to add rules for shell scripts and remove rules
for ReST files that are no longer needed after switch to Markdown.

BREAKING CHANGE: Refactors `run-tests.sh` script options.

.commitlintrc.yaml

@@ -0,0 +1,21 @@
+rules:
+  body-case: [2, always, sentence-case]
+  body-full-stop: [1, always]
+  body-leading-blank: [2, always]
+  body-max-line-length: [2, always, 72]
+  footer-leading-blank: [2, always]
+  footer-max-line-length: [2, always, 72]
+  header-max-length: [2, always, 72]
+  scope-case: [2, always, lower-case]
+  subject-case:
+    - 2
+    - never
+    - [pascal-case, sentence-case, start-case, upper-case]
+  subject-empty: [2, never]
+  subject-full-stop: [2, never, "."]
+  type-case: [2, always, lower-case]
+  type-empty: [2, never]
+  type-enum:
+    - 2
+    - always
+    - [build, chore, ci, docs, feat, fix, perf, refactor, style, test]

.editorconfig

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2016 CERN.
+# Copyright (C) 2016, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -40,8 +40,8 @@ known_first_party = cernopendata
 multi_line_output = 2
 default_section = THIRDPARTY
 
-# RST files (used by sphinx)
-[*.rst]
+# Shell script files
+[*.sh]
 indent_size = 4
 
 # CSS, HTML, JS, JSON, YML

.flake8

@@ -0,0 +1,19 @@
+[flake8]
+max-line-length = 89
+
+exclude =
+    build
+    dist
+    docs
+    coverage.xml
+    cernopendata_client.egg-info
+    .*/
+    env/
+    .git
+    __pycache__
+
+ignore = E203, E231, E266, E501, W503, F403, F401, W605
+
+max-complexity = 18
+
+select = B,C,E,F,W,T4,B9

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 # This file is part of CERN Open Data Portal.
-# Copyright (C) 2020, 2023, 2024 CERN.
+# Copyright (C) 2020, 2023, 2024, 2025 CERN.
 #
 # CERN Open Data Portal is free software; you can redistribute it
 # and/or modify it under the terms of the GNU General Public License as
@@ -20,117 +20,241 @@
 # waive the privileges and immunities granted to it by virtue of its status
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
-name: CI
+name: ci
 
 on: [push, pull_request]
 
 jobs:
-  lint-shellcheck:
-    runs-on: ubuntu-20.04
+  data-dois:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check data DOIs
+        run: ./run-tests.sh --data-dois
+
+  data-json:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Runs shell script static analysis
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Check data JSON
         run: |
-          sudo apt-get install shellcheck
-          ./run-tests.sh --check-shellscript
+          pip install click
+          ./run-tests.sh --data-json
 
-  lint-black:
-    runs-on: ubuntu-20.04
+  data-licenses:
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check Python code formatting
+      - name: Check data licenses
         run: |
-          pip install black
-          ./run-tests.sh --check-black
+          ./run-tests.sh --data-licenses
+
+  data-recids:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
 
-  lint-pycodestyle:
-    runs-on: ubuntu-20.04
+      - name: Check data record IDs
+        run: ./run-tests.sh --data-recids
+
+  data-slugs:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check data slugs
+        run: ./run-tests.sh --data-slugs
+
+  data-types:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+
+      - name: Check data types
+        run: ./run-tests.sh --data-types
+
+  format-black:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check compliance with Python coding style conventions
+      - name: Check formatting of Python code
         run: |
-          pip install --upgrade pip
-          pip install pycodestyle
-          ./run-tests.sh --check-pycodestyle
+          pip install black
+          ./run-tests.sh --format-black
 
-  lint-pydocstyle:
-    runs-on: ubuntu-20.04
+  format-isort:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check compliance with Python docstring conventions
+      - name: Check formatting of Python imports
         run: |
-          pip install --upgrade pip
-          pip install pydocstyle
-          ./run-tests.sh --check-pydocstyle
+          pip install isort
+          ./run-tests.sh --format-isort
+
+  format-shfmt:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check formatting of shell scripts
+        run: |
+          sudo apt-get install shfmt
+          ./run-tests.sh --format-shfmt
 
-  check-fixtures:
-    runs-on: ubuntu-20.04
+  format-whitespace:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4
+
+      - name: Check formatting of whitespace
+        run: ./run-tests.sh --format-whitespace
+
+  lint-commitlint:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
-          node-version: "14"
+          fetch-depth: 0
 
-      - name: Install Node dependencies
-        run: npm install -g jsonlint
+      - name: Setup Node
+        uses: actions/setup-node@v4
 
-      - name: Check fixtures
-        run: ./run-tests.sh --check-fixtures
+      - name: Install commitlint
+        run: |
+          npm install conventional-changelog-conventionalcommits
+          npm install commitlint@latest
+
+      - name: Check commit message compliance of the recently pushed commit
+        if: github.event_name == 'push'
+        run: |
+          ./run-tests.sh --lint-commitlint HEAD~1 HEAD
 
-  check-isort:
-    runs-on: ubuntu-20.04
+      - name: Check commit message compliance of the pull request
+        if: github.event_name == 'pull_request'
+        run: |
+          ./run-tests.sh --lint-commitlint ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.number }}
+
+  lint-flake8:
+    runs-on: ubuntu-24.04
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check isort
+      - name: Check linting of Python code
         run: |
-          pip install --upgrade pip
-          pip install isort
-          ./run-tests.sh --check-isort
+          pip install flake8
+          ./run-tests.sh --lint-flake8
+
+  lint-jsonlint:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+
+      - name: Check linting of JSON files
+        run: |
+          npm install jsonlint --global
+          ./run-tests.sh --lint-jsonlint
 
-  check-licenses:
-    runs-on: ubuntu-20.04
+  lint-pydocstyle:
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - name: Check licenses
+      - name: Check linting of Python docstrings
+        run: |
+          pip install pydocstyle
+          ./run-tests.sh --lint-pydocstyle
+
+  lint-shellcheck:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check linting of shell scripts
+        run: |
+          sudo apt-get install shellcheck
+          ./run-tests.sh --lint-shellcheck
+
+  lint-yamllint:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Lint YAML files
         run: |
-          pip install --upgrade pip
-          ./run-tests.sh --check-licenses
+          pip install yamllint
+          ./run-tests.sh --lint-yamllint

.yamllint.yaml

@@ -0,0 +1,8 @@
+extends: default
+
+rules:
+  comments:
+    min-spaces-from-content: 1
+  document-start: disable
+  line-length: disable
+  truthy: disable

data/docs/atlas-heavy-ion-open-data-for-research-release-2024/atlas-heavy-ion-open-data-for-research-release-2024.json

@@ -6,7 +6,9 @@
       "format": "md"
     },
     "date_published": "2024-12-13",
-    "experiment": ["ATLAS"],
+    "experiment": [
+      "ATLAS"
+    ],
     "featured": 1,
     "short_description": {
       "content": "The ATLAS Collaboration has released its first open data of heavy-ion collisions for research purposes. This dataset features lead-ion (Pb-Pb) collisions at an energy of 5 TeV per nucleon pair, recorded in 2015 as part of the Large Hadron Collider’s second operation period (LHC Run 2)."

data/docs/lhcb-about/lhcb-about.json

@@ -4,7 +4,6 @@
       "content": "lhcb-about.md",
       "format": "md"
     },
-
     "experiment": [
       "LHCb"
     ],