Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions .github/workflows/test-mlc-script-features.yml
Original file line number Diff line number Diff line change
Expand Up @@ -695,3 +695,87 @@ jobs:
# detect-cpu depends on detect-os; we can pass adr options through
mlcr detect,cpu --adr.detect-os.tags=detect,os --quiet
echo "PASS: adr option correctly passed through to dependencies"

- name: Create test scripts for adr opt-out behavior
shell: bash
run: |
# Determine the MLC repo path (mlc pull repo clones to ~/MLC/repos/<owner>@<repo>)
# by parsing the output of `mlc find script --tags=detect,os`.
REPO_SCRIPT_DIR=$(python3 - <<'PY'
import os
import re
import subprocess
import sys

out = subprocess.run(
['mlc', 'find', 'script', '--tags=detect,os', '--quiet'],
capture_output=True, text=True)
combined = out.stdout + out.stderr
match = re.search(r'Item path:\s*(.+)', combined)
if not match:
print('ERROR: Could not find script directory from mlc find output', file=sys.stderr)
sys.exit(1)
print(os.path.dirname(match.group(1).strip()))
PY
)
echo "MLC repo script dir: $REPO_SCRIPT_DIR"

mkdir -p "$REPO_SCRIPT_DIR/test-adr-recursive-local-child"
cat <<'EOF' > "$REPO_SCRIPT_DIR/test-adr-recursive-local-child/meta.yaml"
alias: test-adr-recursive-local-child
automation_alias: script
automation_uid: 5b4e0237da074764
deps:
- names:
- mydep
tags: detect,os
skip_add_deps_recursive: true
tags:
- test
- adr-recursive-local-child
uid: cccc1111dddd2222
EOF
printf '#!/bin/bash\necho "Child script runs"\n' > "$REPO_SCRIPT_DIR/test-adr-recursive-local-child/run.sh"
chmod +x "$REPO_SCRIPT_DIR/test-adr-recursive-local-child/run.sh"
printf '@echo off\necho Child script runs\n' > "$REPO_SCRIPT_DIR/test-adr-recursive-local-child/run.bat"

mkdir -p "$REPO_SCRIPT_DIR/test-adr-recursive-local-parent"
cat <<'EOF' > "$REPO_SCRIPT_DIR/test-adr-recursive-local-parent/meta.yaml"
alias: test-adr-recursive-local-parent
automation_alias: script
automation_uid: 5b4e0237da074764
deps:
- tags: test,adr-recursive-local-child
tags:
- test
- adr-recursive-local-parent
uid: cccc2222dddd3333
EOF
printf '#!/bin/bash\necho "Parent script runs"\n' > "$REPO_SCRIPT_DIR/test-adr-recursive-local-parent/run.sh"
chmod +x "$REPO_SCRIPT_DIR/test-adr-recursive-local-parent/run.sh"
printf '@echo off\necho Parent script runs\n' > "$REPO_SCRIPT_DIR/test-adr-recursive-local-parent/run.bat"

- name: Test skip_add_deps_recursive option
shell: bash
run: |
# Global adr override should not modify child dep marked with skip_add_deps_recursive: true
mlcr test,adr-recursive-local-parent --adr.mydep.tags=nonexistent,script,tags,xxxyyyzzz --print_deps --json --quiet > /tmp/mlc-adr-local-test.json 2>&1
python3 - <<'PY'
import json
import sys

with open('/tmp/mlc-adr-local-test.json', 'r') as f:
output = json.load(f)

dep_tags = [dep.get('tags', '') for dep in output.get('deps', [])]

if not any({'detect', 'os'}.issubset(set(tags.split(','))) for tags in dep_tags):
print(f"ERROR: Expected detect,os dependency tags in resolved deps. Got: {dep_tags}")
sys.exit(1)

if any('nonexistent' in tags.split(',') for tags in dep_tags):
print(f"ERROR: Recursive override leaked into local-only dependency. Got: {dep_tags}")
sys.exit(1)

print("PASS: skip_add_deps_recursive kept recursive adr override local-safe")
PY
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ develop-eggs/
dist/
eggs/
.eggs/
*.egg-info/
lib/
lib64/
sdist/
Expand Down
1 change: 1 addition & 0 deletions automation/script/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,7 @@ deps:
- 'yes'
force_env_keys: # Force these keys into the dep's env
- MLC_DOWNLOAD_*
skip_add_deps_recursive: true # Do not let parent adr overrides change this dep
dynamic: true # Always re-evaluate, even from cache
```

Expand Down
47 changes: 34 additions & 13 deletions automation/script/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -2658,7 +2658,7 @@ def search(self, i):

script_tags = i.get('script_tags', [])
variation_tags = i.get('variation_tags', [])

if not script_tags and tags_string:
r = get_variation_and_script_tags(tags_string.strip())
script_tags = r['script_tags']
Expand Down Expand Up @@ -4483,8 +4483,8 @@ def docker(self, i):
from script.docker import docker_run
return docker_run(self, i)


############################################################

def apptainerfile(self, i):
from script.apptainer import apptainerfile
return apptainerfile(self, i)
Expand All @@ -4494,6 +4494,7 @@ def apptainer(self, i):
from script.apptainer import apptainer_run
return apptainer_run(self, i)
############################################################

def experiment(self, i):
from script.experiment import experiment_run
return experiment_run(self, i)
Expand Down Expand Up @@ -5524,6 +5525,23 @@ def update_deps(deps, add_deps, fail_error=False, env={}):
return {'return': 0}


##############################################################################
def update_deps_recursive(deps, add_deps, env=None):
"""
Internal: apply recursive dependency overrides while skipping deps marked
with skip_add_deps_recursive.
"""
if env is None:
env = {}

recursive_updatable_deps = [
dep for dep in deps
if not is_true(dep.get('skip_add_deps_recursive', False))
]

return update_deps(recursive_updatable_deps, add_deps, False, env)


##############################################################################
def append_deps(deps, new_deps):
"""
Expand Down Expand Up @@ -5619,17 +5637,21 @@ def update_deps_from_input(deps, post_deps, prehook_deps, posthook_deps, i):
return r1

if add_deps_recursive_info_from_input:
update_deps(deps, add_deps_recursive_info_from_input, False, env)
update_deps(post_deps, add_deps_recursive_info_from_input, False, env)
update_deps(
update_deps_recursive(
deps,
add_deps_recursive_info_from_input,
env)
update_deps_recursive(
post_deps,
add_deps_recursive_info_from_input,
env)
update_deps_recursive(
prehook_deps,
add_deps_recursive_info_from_input,
False,
env)
update_deps(
update_deps_recursive(
posthook_deps,
add_deps_recursive_info_from_input,
False,
env)

return {'return': 0}
Expand Down Expand Up @@ -5928,7 +5950,6 @@ def update_state_from_meta(meta, env, state, const, const_state, run_state, i):
if folder_path_env_keys:
run_state['folder_path_env_keys'] += folder_path_env_keys


return {'return': 0}

##############################################################################
Expand All @@ -5940,10 +5961,10 @@ def update_adr_from_meta(deps, post_deps, prehook_deps,
Internal: update add_deps_recursive from meta
"""
if add_deps_recursive_info:
update_deps(deps, add_deps_recursive_info, False, env)
update_deps(post_deps, add_deps_recursive_info, False, env)
update_deps(prehook_deps, add_deps_recursive_info, False, env)
update_deps(posthook_deps, add_deps_recursive_info, False, env)
update_deps_recursive(deps, add_deps_recursive_info, env)
update_deps_recursive(post_deps, add_deps_recursive_info, env)
update_deps_recursive(prehook_deps, add_deps_recursive_info, env)
update_deps_recursive(posthook_deps, add_deps_recursive_info, env)

return {'return': 0}

Expand Down
1 change: 1 addition & 0 deletions script/install-python-venv/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ new_state_keys:
deps:
- inherit_variation_tags: true
reuse_version: true
skip_add_deps_recursive: true
tags: get,python,-virtual
post_deps:
- names:
Expand Down
Loading