From ffb17096300c9c822d9fb80337a576048e96d635 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 2 Dec 2024 11:19:22 +0000 Subject: [PATCH 1/8] Begin v0.3.20dev --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f545597b4..d15a7b5c8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +In progress (unreleased) +------------------------ + + + v0.3.19 (2024-12-02) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index fa9b176de..18932405f 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 19) +__version__ = (0, 3, 20, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From 53b4881628a3f08eb9488b087fbf6ca8d7357674 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 9 Dec 2024 11:44:39 +0000 Subject: [PATCH 2/8] ansible_mitogen: Fix wait_for_connection + templated ansible_python_interpreter This tightens up our monkey patching `Connection._action` so it's only applied during `meta: reset_connection` & promptly removed. This fixes "'int' object has no attribute 'template'" when `ansible.plugins.action.wait_for_connection` or other code calls `ansible.plugins.connection.ConnectionBase.reset()`. This could also have switched to `templar=templar` on the temporary action, rather than `templar=0`, but it's not strictly necessary to fix this bug. I anticipate other changes doing so soon, to improve interpreter discovery & templated python interpreter path support. --- ansible_mitogen/connection.py | 41 ++++++++++++------- docs/changelog.rst | 2 + tests/ansible/hosts/default.hosts | 3 ++ tests/ansible/regression/all.yml | 1 + ...ssue_1079__wait_for_connection_timeout.yml | 10 +++++ tests/ansible/templates/test-targets.j2 | 7 ++++ 6 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 1231acb5b..a043ef1df 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -933,31 +933,39 @@ def reset(self): self.reset_compat_msg ) - # Strategy's _execute_meta doesn't have an action obj but we'll need one for - # running interpreter_discovery - # will create a new temporary action obj for this purpose - self._action = ansible_mitogen.mixins.ActionModuleMixin( - task=0, - connection=self, - play_context=self._play_context, - loader=0, - templar=0, - shared_loader_obj=0 - ) - - # Workaround for https://github.com/ansible/ansible/issues/84238 + # Handle templated connection variables during `meta: reset_connection`. + # Many bugs/implementation details of Mitogen & Ansible collide here. + # See #1079, #1096, #1132, ansible/ansible#84238, ... try: task, templar = self._play_context.vars.pop( '_mitogen.smuggled.reset_connection', ) except KeyError: - pass + self._action_monkey_patched_by_mitogen = False else: + # LOG.info('%r.reset(): remote_addr=%r', self, self._play_context.remote_addr) + # ansible.plugins.strategy.StrategyBase._execute_meta() doesn't + # have an action object, which we need for interpreter_discovery. + # Create a temporary action object for this purpose. + self._action = ansible_mitogen.mixins.ActionModuleMixin( + task=task, + connection=self, + play_context=self._play_context, + loader=templar._loader, + templar=templar, + shared_loader_obj=0, + ) + self._action_monkey_patched_by_mitogen = True + + # Workaround for https://github.com/ansible/ansible/issues/84238 self.set_options( task_keys=task.dump_attrs(), var_options=self._mitogen_var_options(templar), ) + del task + del templar + # Clear out state in case we were ever connected. self.close() @@ -977,6 +985,11 @@ def reset(self): finally: binding.close() + # Cleanup any monkey patching we did for `meta: reset_connection` + if self._action_monkey_patched_by_mitogen: + del self._action + del self._action_monkey_patched_by_mitogen + # Compatibility with Ansible 2.4 wait_for_connection plug-in. _reset = reset diff --git a/docs/changelog.rst b/docs/changelog.rst index d15a7b5c8..0b83a04cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection` + timeout with templated ``ansible_python_interpreter`` v0.3.19 (2024-12-02) diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 17d1fd6d6..a232c6ab0 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -19,6 +19,9 @@ ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}" ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ inventory_hostname }} +[issue1079] +wait-for-connection ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" + [tt_targets_bare] tt-bare diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index a4272805f..a7c8033e4 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -16,4 +16,5 @@ - import_playbook: issue_776__load_plugins_called_twice.yml - import_playbook: issue_952__ask_become_pass.yml - import_playbook: issue_1066__add_host__host_key_checking.yml +- import_playbook: issue_1079__wait_for_connection_timeout.yml - import_playbook: issue_1087__template_streamerror.yml diff --git a/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml new file mode 100644 index 000000000..2ff1899bd --- /dev/null +++ b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml @@ -0,0 +1,10 @@ +- name: regression/issue_1079__wait_for_connection_timeout.yml + hosts: issue1079 + gather_facts: false + tasks: + - name: Wait for connection at start of play + wait_for_connection: + timeout: 5 + tags: + - issue_1079 + - wait_for_connection diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index bb0d85ece..87b0e7c90 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -40,6 +40,13 @@ ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ '{{' }} inventory_hostname {{ '}}' {% set tt = containers[0] %} +[issue1079] +wait-for-connection ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}" + +[issue1079:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password + [tt_targets_bare] tt-bare From d033f7b057f55402613c516b80e31d18c57bc07d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Dec 2024 15:28:01 +0000 Subject: [PATCH 3/8] ansible_mitogen: Restore dummy objects in Connection.reset() The previous commit (53b4881628a3f08eb9488b087fbf6ca8d7357674 in PR 1200) was not intended to change these values, but some WIP slipped through. This partially reverts that commit so the two changes (moving the monkey patch, making the monkey patch more capable) exist in distinct commits. --- ansible_mitogen/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a043ef1df..90ddb7ed7 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -948,11 +948,11 @@ def reset(self): # have an action object, which we need for interpreter_discovery. # Create a temporary action object for this purpose. self._action = ansible_mitogen.mixins.ActionModuleMixin( - task=task, + task=0, connection=self, play_context=self._play_context, - loader=templar._loader, - templar=templar, + loader=0, + templar=0, shared_loader_obj=0, ) self._action_monkey_patched_by_mitogen = True From 941da317357ad343a758a1a1a4a2de10b4ad5826 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Dec 2024 15:50:57 +0000 Subject: [PATCH 4/8] build: Reduce macOS job timeout to mitigate BlockingIOError This will mitigate the impact of #1185 a little, which causes the job to continue running without making progress, until it hits this timeout. Successful jobs are typically completing in 8 - 12 minutes. refs #1185 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 24b70c64b..9a275a00c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -162,7 +162,7 @@ jobs: macos: # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md runs-on: macos-13 - timeout-minutes: 120 + timeout-minutes: 15 strategy: fail-fast: false From 6900e88dfdf4e77d4800dae8b48bf82e9b7ad419 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Dec 2024 16:15:06 +0000 Subject: [PATCH 5/8] ansible_mitogen: Fix templated python interpreter with `meta: reset_connection` refs #1079 --- ansible_mitogen/connection.py | 6 +++--- docs/changelog.rst | 2 ++ .../issue_1079__wait_for_connection_timeout.yml | 12 ++++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 90ddb7ed7..a043ef1df 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -948,11 +948,11 @@ def reset(self): # have an action object, which we need for interpreter_discovery. # Create a temporary action object for this purpose. self._action = ansible_mitogen.mixins.ActionModuleMixin( - task=0, + task=task, connection=self, play_context=self._play_context, - loader=0, - templar=0, + loader=templar._loader, + templar=templar, shared_loader_obj=0, ) self._action_monkey_patched_by_mitogen = True diff --git a/docs/changelog.rst b/docs/changelog.rst index 0b83a04cc..9add8ba09 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,8 @@ In progress (unreleased) * :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection` timeout with templated ``ansible_python_interpreter`` +* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix templated python interpreter + with `meta: reset_connection` v0.3.19 (2024-12-02) diff --git a/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml index 2ff1899bd..6ded03ea2 100644 --- a/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml +++ b/tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml @@ -8,3 +8,15 @@ tags: - issue_1079 - wait_for_connection + +- hosts: issue1079 + gather_facts: false + tasks: + - meta: reset_connection + - name: Wait for connection after reset_connection + wait_for_connection: + timeout: 5 + tags: + - issue_1079 + - reset_connection + - wait_for_connection From 5e6d7bf4fb348b01889e111323c4c18211a684e9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Dec 2024 17:28:46 +0000 Subject: [PATCH 6/8] ansible_mitogen: Templated connection timeout Ansible >= 4 (ansible-core >= 2.11) the SSH plugin has a `timeout` option and with variable `ansible_ssh_timeout`, but not a `ansible_timeout` variable. The local plugin has no such option or variable(s). However `ansible_timeout` is backfilled for all conection plugins, by legacy mechanisms that populate the play context attribute: - `ansible.constants.COMMON_CONNECTION_VARS` - `ansible.constants.MAGIC_VARIABLE_MAPPING` The `timeout` keyword is for task completion timeout, not connection timeout. --- ansible_mitogen/connection.py | 16 ++++++------ ansible_mitogen/strategy.py | 26 ++++++++++++++++--- ansible_mitogen/transport_config.py | 8 ++---- docs/changelog.rst | 2 ++ tests/ansible/hosts/default.hosts | 1 + .../ssh/templated_by_play_taskvar.yml | 2 ++ ..._109__target_has_old_ansible_installed.yml | 1 + tests/ansible/templates/test-targets.j2 | 1 + 8 files changed, 40 insertions(+), 17 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a043ef1df..5053a5f5b 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -145,7 +145,7 @@ def _connect_ssh(spec): 'identity_file': private_key_file, 'identities_only': False, 'ssh_path': spec.ssh_executable(), - 'connect_timeout': spec.ansible_ssh_timeout(), + 'connect_timeout': spec.timeout(), 'ssh_args': spec.ssh_args(), 'ssh_debug_level': spec.mitogen_ssh_debug_level(), 'remote_name': get_remote_name(spec), @@ -169,7 +169,7 @@ def _connect_buildah(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -185,7 +185,7 @@ def _connect_docker(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(rediscover_python=True), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -200,7 +200,7 @@ def _connect_kubectl(spec): 'kwargs': { 'pod': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'kubectl_path': spec.mitogen_kubectl_path(), 'kubectl_args': spec.extra_args(), 'remote_name': get_remote_name(spec), @@ -218,7 +218,7 @@ def _connect_jail(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -234,7 +234,7 @@ def _connect_lxc(spec): 'container': spec.remote_addr(), 'python_path': spec.python_path(), 'lxc_attach_path': spec.mitogen_lxc_attach_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -250,7 +250,7 @@ def _connect_lxd(spec): 'container': spec.remote_addr(), 'python_path': spec.python_path(), 'lxc_path': spec.mitogen_lxc_path(), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } @@ -273,7 +273,7 @@ def _connect_podman(spec): 'username': spec.remote_user(), 'container': spec.remote_addr(), 'python_path': spec.python_path(rediscover_python=True), - 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'connect_timeout': spec.timeout(), 'remote_name': get_remote_name(spec), } } diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index c319f3e11..440e58112 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -328,8 +328,12 @@ def run(self, iterator, play_context, result=0): finally: ansible_mitogen.process.set_worker_model(None) - def _smuggle_to_connction_reset(self, task, play_context, iterator, target_host): - # Workaround for https://github.com/ansible/ansible/issues/84238 + def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host): + """ + Create a templar and make it available for use in Connection.reset(). + This allows templated connection variables to be used when Mitogen + reconstructs its connection stack. + """ variables = self._variable_manager.get_vars( play=iterator._play, host=target_host, task=task, _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all, @@ -337,13 +341,29 @@ def _smuggle_to_connction_reset(self, task, play_context, iterator, target_host) templar = ansible.template.Templar( loader=self._loader, variables=variables, ) + + # Required for remote_user option set by variable (e.g. ansible_user). + # Without it remote_user in ansible.cfg gets used. + play_context = play_context.set_task_and_variable_override( + task=task, variables=variables, templar=templar, + ) + play_context.post_validate(templar=templar) + + # Required for timeout option set by variable (e.g. ansible_timeout). + # Without it the task timeout keyword (default: 0) gets used. + play_context.update_vars(variables) + + # Stash the task and templar somewhere Connection.reset() can find it play_context.vars.update({ '_mitogen.smuggled.reset_connection': (task, templar), }) + return play_context def _execute_meta(self, task, play_context, iterator, target_host): if task.args['_raw_params'] == 'reset_connection': - self._smuggle_to_connction_reset(task, play_context, iterator, target_host) + play_context = self._smuggle_to_connection_reset( + task, play_context, iterator, target_host, + ) return super(StrategyMixin, self)._execute_meta( task, play_context, iterator, target_host, diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 2218a7fa7..937755528 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -513,14 +513,10 @@ def ssh_executable(self): return self._connection_option('ssh_executable') def timeout(self): - return self._play_context.timeout + return self._connection_option('timeout') def ansible_ssh_timeout(self): - return ( - self._connection.get_task_var('ansible_timeout') or - self._connection.get_task_var('ansible_ssh_timeout') or - self.timeout() - ) + return self.timeout() def ssh_args(self): return [ diff --git a/docs/changelog.rst b/docs/changelog.rst index 9add8ba09..23d3b327d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,8 @@ In progress (unreleased) timeout with templated ``ansible_python_interpreter`` * :gh:issue:`1079` :mod:`ansible_mitogen`: Fix templated python interpreter with `meta: reset_connection` +* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated connection timeout + (e.g. ``ansible_timeout``). v0.3.19 (2024-12-02) diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index a232c6ab0..3d20e73df 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -52,3 +52,4 @@ tt-port ansible_host=localhost ansible_password=has_sudo_ tt-private-key-file ansible_host=localhost ansible_private_key_file="{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_user=mitogen__has_sudo_pubkey tt-remote-user ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}" tt-ssh-executable ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_ssh_executable="{{ 'ssh' | trim }}" ansible_user=mitogen__has_sudo_nopw +tt-timeout ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_timeout="{{ 5 | int }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml index 4d7e318e9..c5c2e5443 100644 --- a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml +++ b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml @@ -7,6 +7,7 @@ ansible_password: "{{ 'has_sudo_nopw_password' | trim }}" ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" ansible_ssh_executable: "{{ 'ssh' | trim }}" + ansible_timeout: "{{ 5 | int }}" ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" tasks: @@ -23,6 +24,7 @@ ansible_private_key_file: "{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" ansible_ssh_executable: "{{ 'ssh' | trim }}" + ansible_timeout: "{{ 5 | int }}" ansible_user: "{{ 'mitogen__has_sudo_pubkey' | trim }}" tasks: diff --git a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml index 064832ece..a7ae0908c 100644 --- a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml +++ b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml @@ -26,6 +26,7 @@ - env.cwd == ansible_user_dir - (not env.mitogen_loaded) or (env.python_path.count("") == 1) fail_msg: | + ansible_user_dir={{ ansible_user_dir }} env={{ env }} - name: Run some new-style from ansible.module_utils... modules diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index 87b0e7c90..65f2fd7d9 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -86,3 +86,4 @@ tt-port ansible_host={{ tt.hostname }} ansible_password=h tt-private-key-file ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_private_key_file="{{ '{{' }} git_basedir {{ '}}' }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_pubkey tt-remote-user ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}" tt-ssh-executable ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_ssh_executable="{{ '{{' }} 'ssh' | trim {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw +tt-timeout ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_timeout="{{ '{{' }} 5 | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw From 945e3603638b3f3674fb3fdcb4d21e6667bc98bc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 3 Jan 2025 08:52:39 +0000 Subject: [PATCH 7/8] ansible_mitogen: Respect interpreter_python and ANSIBLE_PYTHON_INTERPRETER This adapts PR #740 by @extmind (afe00268906fa5b84b5d94273dd87d60385f8905), which augmented the call to `Connection.get_task_var()` with `C.config.get_config_value('INTERPRETER_PYTHON'` as a default. Instead this *replaces* the call to `Connection.get_task_var()`. The aim is greater simplicity by disentangling templating of a configured interpreter path and discovery of an interpreter when none is configured. I think this also reduces the number of times `Connection._get_task_vars()` is called, so reducing the number of times we do the ugly stack frame inspection. I've also added test cases. Co-authored-by: Lars Beckers --- ansible_mitogen/transport_config.py | 22 +++++++++++++++++----- docs/changelog.rst | 2 ++ tests/ansible/hosts/default.hosts | 1 + tests/ansible/templates/test-targets.j2 | 1 + 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 937755528..effb4d62e 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -493,12 +493,24 @@ def port(self): return self._connection_option('port') def python_path(self, rediscover_python=False): - s = self._connection.get_task_var('ansible_python_interpreter') - # #511, #536: executor/module_common.py::_get_shebang() hard-wires - # "/usr/bin/python" as the default interpreter path if no other - # interpreter is specified. + # See also + # - ansible_mitogen.connecton.Connection.get_task_var() + try: + delegated_vars = self._task_vars['ansible_delegated_vars'] + variables = delegated_vars[self._connection.delegate_to_hostname] + except KeyError: + variables = self._task_vars + + interpreter_python = C.config.get_config_value( + 'INTERPRETER_PYTHON', variables=variables, + ) + + if '{{' in interpreter_python or '{%' in interpreter_python: + templar = self._connection.templar + interpreter_python = templar.template(interpreter_python) + return parse_python_path( - s, + interpreter_python, task_vars=self._task_vars, action=self._action, rediscover_python=rediscover_python) diff --git a/docs/changelog.rst b/docs/changelog.rst index 23d3b327d..13a698fcd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,8 @@ In progress (unreleased) with `meta: reset_connection` * :gh:issue:`1083` :mod:`ansible_mitogen`: Templated connection timeout (e.g. ``ansible_timeout``). +* :gh:issue:`740` :mod:`ansible_mitogen`: Respect ``interpreter_python`` + in ``ansible.cfg`` and ``ANSIBLE_PYTHON_INTERPRETER`` environment variable. v0.3.19 (2024-12-02) diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 3d20e73df..a99352b36 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -50,6 +50,7 @@ tt-host-key-checking ansible_host=localhost ansible_host_key_checking= tt-password ansible_host=localhost ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw tt-port ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw tt-private-key-file ansible_host=localhost ansible_private_key_file="{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_user=mitogen__has_sudo_pubkey +tt-python-interpreter ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_python_interpreter="{{ ansible_playbook_python | trim }}" ansible_user=mitogen__has_sudo_nopw tt-remote-user ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}" tt-ssh-executable ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_ssh_executable="{{ 'ssh' | trim }}" ansible_user=mitogen__has_sudo_nopw tt-timeout ansible_host=localhost ansible_password=has_sudo_nopw_password ansible_timeout="{{ 5 | int }}" ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index 65f2fd7d9..063847ccb 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -84,6 +84,7 @@ tt-host-key-checking ansible_host={{ tt.hostname }} ansible_host_key_c tt-password ansible_host={{ tt.hostname }} ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw tt-port ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_nopw tt-private-key-file ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_private_key_file="{{ '{{' }} git_basedir {{ '}}' }}/tests/data/docker/mitogen__has_sudo_pubkey.key" ansible_python_interpreter={{ tt.python_path }} ansible_user=mitogen__has_sudo_pubkey +tt-python-interpreter ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_password=has_sudo_nopw_password ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw tt-remote-user ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}" tt-ssh-executable ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_ssh_executable="{{ '{{' }} 'ssh' | trim {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw tt-timeout ansible_host={{ tt.hostname }} ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} ansible_timeout="{{ '{{' }} 5 | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw From f1f8ad11ee84f6994ad6a4f96e4b4a5c0c9cc106 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 7 Jan 2025 09:14:31 +0000 Subject: [PATCH 8/8] Prepare v0.3.20 --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 13a698fcd..653e7971f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -In progress (unreleased) ------------------------- +v0.3.20 (2025-01-07) +-------------------- * :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection` timeout with templated ``ansible_python_interpreter`` diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 18932405f..08a3bc58a 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ #: Library version as a tuple. -__version__ = (0, 3, 20, 'dev') +__version__ = (0, 3, 20) #: This is :data:`False` in slave contexts. Previously it was used to prevent