Merge branch 'master' into become_chdir

Author: moreati

.ci/README.md

@@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
 * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
   name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
 * `DISTROS`: the `ansible_` tests can run against multiple targets

.ci/ansible_install.py

@@ -6,11 +6,13 @@
 import os
 import signal
 import sys
-import textwrap
+
+import jinja2
 
 import ci_lib
 
 
+TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
 TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
 HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
 
@@ -33,7 +35,7 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers()
+    containers = ci_lib.container_specs(ci_lib.DISTROS)
     ci_lib.start_containers(containers)
 
 
@@ -52,37 +54,19 @@ def pause_if_interactive():
         distros[container['distro']].append(container['name'])
         families[container['family']].append(container['name'])
 
+    jinja_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
+        lstrip_blocks=True,  # Remove spaces and tabs from before a block
+        trim_blocks=True,  # Remove first newline after a block
+    )
+    inventory_template = jinja_env.get_template('test-targets.j2')
     inventory_path = os.path.join(HOSTS_DIR, 'target')
+
     with open(inventory_path, 'w') as fp:
-        fp.write('[test-targets]\n')
-        fp.writelines(
-            "%(name)s "
-            "ansible_host=%(hostname)s "
-            "ansible_port=%(port)s "
-            "ansible_python_interpreter=%(python_path)s "
-            "ansible_user=mitogen__has_sudo_nopw "
-            "ansible_password=has_sudo_nopw_password"
-            "\n"
-            % container
-            for container in containers
-        )
-
-        for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]):
-            fp.write('\n[%s]\n' % distro)
-            fp.writelines('%s\n' % name for name in hostnames)
-
-        for family, hostnames in sorted(families.items(), key=lambda t: t[0]):
-            fp.write('\n[%s]\n' % family)
-            fp.writelines('%s\n' % name for name in hostnames)
-
-        fp.write(textwrap.dedent(
-            '''
-            [linux:children]
-            test-targets
-
-            [linux_containers:children]
-            test-targets
-            '''
+        fp.write(inventory_template.render(
+            containers=containers,
+            distros=distros,
+            families=families,
         ))
 
     ci_lib.dump_file(inventory_path)

.ci/ansible_tests.py

@@ -14,6 +14,19 @@ steps:
     versionSpec: '$(python.version)'
   condition: ne(variables['python.version'], '')
 
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws
+  displayName: Authenticate to container registry
+  condition: eq(variables['Agent.OS'], 'Linux')
+  env:
+    AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
+    AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
+
 - script: |
     set -o errexit
     set -o nounset
@@ -90,7 +103,3 @@ steps:
 
     "$PYTHON" -m tox -e "$(tox.env)"
   displayName: "Run tests"
-  env:
-    AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
-    AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
-    AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

.ci/azure-pipelines-steps.yml

@@ -16,37 +16,26 @@ trigger:
       - docs-master
 
 jobs:
-- job: mac11
+- job: mac12
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
-    # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md
-    vmImage: macOS-11
+    # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
+    vmImage: macOS-12
   strategy:
     matrix:
-      Mito_27:
-        tox.env: py27-mode_mitogen
       Mito_312:
-        python.version: '3.12'
         tox.env: py312-mode_mitogen
-
-      Loc_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
-      Loc_312_9:
-        python.version: '3.12'
-        tox.env: py312-mode_localhost-ansible9
-
-      Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
-      Van_312_9:
-        python.version: '3.12'
-        tox.env: py312-mode_localhost-ansible9-strategy_linear
+      Loc_312_10:
+        tox.env: py312-mode_localhost-ansible10
+      Van_312_10:
+        tox.env: py312-mode_localhost-ansible10-strategy_linear
 
 - job: Linux
   pool:
-    # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
     vmImage: ubuntu-20.04
   steps:
   - template: azure-pipelines-steps.yml
@@ -163,3 +152,6 @@ jobs:
       Ans_312_9:
         python.version: '3.12'
         tox.env: py312-mode_ansible-ansible9
+      Ans_312_10:
+        python.version: '3.12'
+        tox.env: py312-mode_ansible-ansible10

.ci/azure-pipelines.yml

@@ -27,6 +27,13 @@
     )
 )
 
+
+IMAGE_TEMPLATE = os.environ.get(
+    'MITOGEN_TEST_IMAGE_TEMPLATE',
+    'public.ecr.aws/n5z0e8q9/%(distro)s-test',
+)
+
+
 _print = print
 def print(*args, **kwargs):
     file = kwargs.get('file', sys.stdout)
@@ -193,8 +200,6 @@ def __exit__(self, _1, _2, _3): pass
 DISTRO = os.environ.get('DISTRO', 'debian9')
 # Used only when MODE=ansible
 DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
-TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
-BASE_PORT = 2200
 TMP = TempDir().path
 
 
@@ -217,6 +222,7 @@ def __exit__(self, _1, _2, _3): pass
 def get_docker_hostname():
     """Return the hostname where the docker daemon is running.
     """
+    # Duplicated in testlib
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -225,27 +231,34 @@ def get_docker_hostname():
     return parsed.netloc.partition(':')[0]
 
 
-def make_containers(name_prefix='', port_offset=0):
+def container_specs(
+        distros,
+        base_port=2200,
+        image_template=IMAGE_TEMPLATE,
+        name_template='target-%(distro)s-%(index)d',
+):
     """
     >>> import pprint
-    >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
-    >>> pprint.pprint(make_containers())
+    >>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
     [{'distro': 'debian11',
       'family': 'debian',
       'hostname': 'localhost',
       'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
+      'index': 1,
       'name': 'target-debian11-1',
       'port': 2201,
-      'python_path': '/usr/bin/python'},
+      'python_path': '/usr/bin/python3'},
      {'distro': 'centos6',
       'family': 'centos',
       'hostname': 'localhost',
       'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
+      'index': 2,
       'name': 'target-centos6-2',
       'port': 2202,
       'python_path': '/usr/bin/python'}]
     """
     docker_hostname = get_docker_hostname()
+    # Code duplicated in testlib.py, both should be updated together
     distro_pattern = re.compile(r'''
         (?P<distro>(?P<family>[a-z]+)[0-9]+)
         (?:-(?P<py>py3))?
@@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
     i = 1
     lst = []
 
-    for distro in DISTROS:
+    for distro in distros:
+        # Code duplicated in testlib.py, both should be updated together
         d = distro_pattern.match(distro).groupdict(default=None)
-        distro = d['distro']
-        family = d['family']
-        image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
 
-        if d['py'] == 'py3':
+        if d.pop('py') == 'py3':
             python_path = '/usr/bin/python3'
         else:
             python_path = '/usr/bin/python'
 
-        if d['count']:
-            count = int(count)
-        else:
-            count = 1
+        count = int(d.pop('count') or '1', 10)
 
         for x in range(count):
-            lst.append({
-                "distro": distro, "family": family, "image": image,
-                "name": name_prefix + ("target-%s-%s" % (distro, i)),
+            d['index'] = i
+            d.update({
+                'image': image_template % d,
+                'name': name_template % d,
                 "hostname": docker_hostname,
-                "port": BASE_PORT + i + port_offset,
+                'port': base_port + i,
                 "python_path": python_path,
             })
+            lst.append(d)
             i += 1
 
     return lst

.ci/ci_lib.py

@@ -2,16 +2,10 @@
 
 import ci_lib
 
-# Naturally DebOps only supports Debian.
-ci_lib.DISTROS = ['debian']
-
 ci_lib.run_batches([
     [
         'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
     ],
-    [
-        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-    ],
 ])
 
 ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

.ci/debops_common_install.py

@@ -6,17 +6,18 @@
 import ci_lib
 
 
-# DebOps only supports Debian.
-ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
-
 project_dir = os.path.join(ci_lib.TMP, 'project')
 vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
 inventory_path = 'ansible/inventory/hosts'
 docker_hostname = ci_lib.get_docker_hostname()
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
+    containers = ci_lib.container_specs(
+        ['debian*2'],
+        base_port=2700,
+        name_template='debops-target-%(distro)s-%(index)d',
+    )
     ci_lib.start_containers(containers)
 
 

.ci/debops_common_tests.py

@@ -3,9 +3,6 @@
 import ci_lib
 
 batches = [
-    [
-        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-    ],
     [
         'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
     ]