Update CI files

Author: pulpbot

.bumpversion.cfg

@@ -28,8 +28,7 @@ RUN pip3 install --upgrade pip setuptools wheel && \
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \
-  pipdeptree && \
+{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
   rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}

.ci/ansible/Containerfile.j2

@@ -5,8 +5,3 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*
 
 tablib!=3.6.0
 # 3.6.0: This release introduced a regression removing the "html" optional dependency.
-
-
-
-# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue.
-pipdeptree<=3.23.1

.ci/assets/ci_constraints.txt

@@ -1,3 +1,3 @@
-bump2version
+bump-my-version
 gitpython
 towncrier

.ci/assets/release_requirements.txt

@@ -13,6 +13,11 @@
 from packaging.version import Version
 import yaml
 
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib
+
 
 CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml"
 
@@ -59,11 +64,11 @@ def to_upper_bound(req):
                 operator = "~="
                 version = Version(spec.version)
                 if version.micro != 0:
-                    max_version = f"{version.major}.{version.minor}.{version.micro-1}"
+                    max_version = f"{version.major}.{version.minor}.{version.micro - 1}"
                 elif version.minor != 0:
-                    max_version = f"{version.major}.{version.minor-1}"
+                    max_version = f"{version.major}.{version.minor - 1}"
                 elif version.major != 0:
-                    max_version = f"{version.major-1}.0"
+                    max_version = f"{version.major - 1}.0"
                 else:
                     return f"# NO BETTER CONSTRAINT: {req}"
                 return f"{requirement.name}{operator}{max_version}"
@@ -100,18 +105,32 @@ def main():
     parser.add_argument("filename", nargs="*")
     args = parser.parse_args()
 
-    with fileinput.input(files=args.filename) as req_file:
-        for line in req_file:
-            if line.strip().startswith("#"):
-                # Shortcut comment only lines
-                print(line.strip())
-            else:
-                req, comment = split_comment(line)
-                if args.upper:
-                    new_req = to_upper_bound(req)
+    modifier = to_upper_bound if args.upper else to_lower_bound
+
+    req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")]
+    pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")]
+    if req_files:
+        with fileinput.input(files=req_files) as req_file:
+            for line in req_file:
+                if line.strip().startswith("#"):
+                    # Shortcut comment only lines
+                    print(line.strip())
                 else:
-                    new_req = to_lower_bound(req)
-                print(new_req + comment)
+                    req, comment = split_comment(line)
+                    new_req = modifier(req)
+                    print(new_req + comment)
+    for filename in pyp_files:
+        with open(filename, "rb") as fp:
+            pyproject = tomllib.load(fp)
+            for req in pyproject["project"]["dependencies"]:
+                new_req = modifier(req)
+                print(new_req)
+            optional_dependencies = pyproject["project"].get("optional-dependencies")
+            if optional_dependencies:
+                for opt in optional_dependencies.values():
+                    for req in opt:
+                        new_req = modifier(req)
+                        print(new_req)
 
 
 if __name__ == "__main__":

.ci/scripts/calc_constraints.py

@@ -1,28 +1,21 @@
 #!/usr/bin/env python
 
-# WARNING: DO NOT EDIT!
-#
-# This file was generated by plugin_template, and is managed by it. Please use
-# './plugin-template --github pulp_npm' to update this file.
-#
-# For more info visit https://github.com/pulp/plugin_template
-
 import argparse
 import re
 import os
+import tomllib
 import yaml
+from pathlib import Path
 from tempfile import TemporaryDirectory
 from packaging.version import Version
 from git import Repo
 
-UPSTREAM_REMOTE = "https://github.com/pulp/pulp_npm.git"
-DEFAULT_BRANCH = "main"
 RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$"
 Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"]
 Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"]
 
 
-def main():
+def options():
     """Check which branches need a release."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -32,17 +25,60 @@ def main():
         "'supported'. Defaults to 'supported', see `supported_release_branches` in "
         "`plugin_template.yml`.",
     )
-    opts = parser.parse_args()
+    return parser.parse_args()
+
+
+def template_config():
+    # Assume this script lies in .ci/scripts
+    path = Path(__file__).absolute().parent.parent.parent / "template_config.yml"
+    return yaml.safe_load(path.read_text())
+
 
+def current_version(repo, commitish):
+    try:
+        pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml"))
+        try:
+            current_version = pyproject_toml["project"]["version"]
+        except Exception:
+            current_version = pyproject_toml["tool"]["bumpversion"]["current_version"]
+    except Exception:
+        current_version = repo.git.grep(
+            "current_version", commitish, "--", ".bumpversion.cfg"
+        ).split("=")[-1]
+    return Version(current_version)
+
+
+def check_pyproject_dependencies(repo, from_commit, to_commit):
+    try:
+        old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml"))
+        old_dependencies = set(old_pyproject["project"]["dependencies"])
+        new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml"))
+        new_dependencies = set(new_pyproject["project"]["dependencies"])
+        if old_dependencies != new_dependencies:
+            return ["dependencies"]
+        else:
+            return []
+    except Exception as e:
+        print(f"WARNING: Comparing the dependencies in pyproject.toml failed. ({e})")
+        # Gathering more details failed.
+        return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."]
+
+
+def main(options, template_config):
     with TemporaryDirectory() as d:
         # Clone from upstream to ensure we have updated branches & main
+        GITHUB_ORG = template_config["github_org"]
+        PLUGIN_NAME = template_config["plugin_name"]
+        UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git"
+        DEFAULT_BRANCH = template_config["plugin_default_branch"]
+
         repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none")
         heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")]
         available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
         available_branches.sort(key=lambda ver: Version(ver))
         available_branches.append(DEFAULT_BRANCH)
 
-        branches = opts.branches
+        branches = options.branches
         if branches == "supported":
             with open(f"{d}/template_config.yml", mode="r") as f:
                 tc = yaml.safe_load(f)
@@ -86,6 +122,11 @@ def main():
                 )
                 if req_txt_diff:
                     reasons.append("requirements.txt")
+                pyproject_diff = repo.git.diff(
+                    f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml"
+                )
+                if pyproject_diff:
+                    reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}"))
 
                 if reasons:
                     curr_version = Version(last_tag)
@@ -106,15 +147,10 @@ def main():
                 for change in changes.split("\n"):
                     _, ext = os.path.splitext(change)
                     if ext in Y_CHANGELOG_EXTS:
-                        # We don't put Y release bumps in the commit message, check file instead
-                        # The 'current_version' is always the next version to release
-                        next_version = repo.git.grep(
-                            "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg"
-                        ).split("=")[-1]
-                        next_version = Version(next_version)
-                        print(
-                            f"A new Y-release is needed! New Version: {next_version.base_version}"
-                        )
+                        # We don't put Y release bumps in the commit message, check file instead.
+                        # The 'current_version' is always the dev of the next version to release.
+                        next_version = current_version(repo, DEFAULT_BRANCH).base_version
+                        print(f"A new Y-release is needed! New Version: {next_version}")
                         releases.append(next_version)
                         break
 
@@ -123,4 +159,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main(options(), template_config())

.ci/scripts/check_release.py

@@ -5,11 +5,13 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+import tomllib
 import warnings
-from pkg_resources import Requirement
+from packaging.requirements import Requirement
 
 
 CHECK_MATRIX = [
+    ("pyproject.toml", True, True, True),
     ("requirements.txt", True, True, True),
     ("dev_requirements.txt", False, True, False),
     ("ci_requirements.txt", False, True, True),
@@ -20,17 +22,33 @@
     ("clitest_requirements.txt", False, True, True),
 ]
 
-errors = []
 
-for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
-    try:
+def iterate_file(filename):
+    if filename == "pyproject.toml":
+        with open(filename, "rb") as fd:
+            pyproject_toml = tomllib.load(fd)
+            if "project" in pyproject_toml:
+                for nr, line in enumerate(pyproject_toml["project"]["dependencies"]):
+                    yield nr, line
+    else:
         with open(filename, "r") as fd:
             for nr, line in enumerate(fd.readlines()):
                 line = line.strip()
                 if not line or line.startswith("#"):
                     continue
+                if "#" in line:
+                    line = line.split("#", maxsplit=1)[0]
+                yield nr, line.strip()
+
+
+def main():
+    errors = []
+
+    for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
+        try:
+            for nr, line in iterate_file(filename):
                 try:
-                    req = Requirement.parse(line)
+                    req = Requirement(line)
                 except ValueError:
                     if line.startswith("git+"):
                         # The single exception...
@@ -49,18 +67,21 @@
                             and req.name != "pulp-npm-client"
                         ):
                             errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.")
-                    ops = [op for op, ver in req.specs]
-                    spec = str(req.specs)
+                    ops = [spec.operator for spec in req.specifier]
                     if "~=" in ops:
                         warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!")
                     elif "<" not in ops and "<=" not in ops and "==" not in ops:
                         if check_upperbound:
                             errors.append(f"{filename}:{nr}: Upper bound missing in {line}.")
-    except FileNotFoundError:
-        # skip this test for plugins that don't use this requirements.txt
-        pass
+        except FileNotFoundError:
+            # skip this test for plugins that don't use this requirements.txt
+            pass
+
+    if errors:
+        print("Dependency issues found:")
+        print("\n".join(errors))
+        exit(1)
+
 
-if errors:
-    print("Dependency issues found:")
-    print("\n".join(errors))
-    exit(1)
+if __name__ == "__main__":
+    main()

.ci/scripts/check_requirements.py

@@ -1 +1 @@
-2021.08.26-387-ge627e91
+2021.08.26-406-g5f397e3

.github/template_gitref

@@ -34,11 +34,11 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install packaging twine wheel mkdocs jq
+          pip install build packaging twine wheel mkdocs jq
           echo ::endgroup::
       - name: "Build package"
         run: |
-          python3 setup.py sdist bdist_wheel --python-tag py3
+          python3 -m build
           twine check dist/*
       - name: "Install built packages"
         run: |

.github/workflows/build.yml

@@ -42,9 +42,6 @@ jobs:
           GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
         run: |
           .github/workflows/scripts/check_commit.sh
-      - name: "Verify requirements files"
-        run: |
-          python .ci/scripts/check_requirements.py
 
   lint:
     uses: "./.github/workflows/lint.yml"
@@ -56,6 +53,9 @@ jobs:
   test:
     needs: "build"
     uses: "./.github/workflows/test.yml"
+    with:
+      matrix_env: |
+        [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}]
 
   deprecations:
     runs-on: "ubuntu-latest"
@@ -84,7 +84,6 @@ jobs:
       - "check-commits"
       - "lint"
       - "test"
-      - "docs"
     if: "always()"
     steps:
       - name: "Collect needed jobs results"