|
| 1 | +#!/usr/bin/env python |
| 2 | +""" |
| 3 | +Lint script to verify that arguments in arguments.py and arguments_typed.py are in sync. |
| 4 | +
|
| 5 | +Due to Python's typing limitations (ParamSpec args cannot be concatenated), we maintain |
| 6 | +two separate definitions of global arguments: |
| 7 | +- arguments.py: TypedDicts (ConnectorArguments, MetaArguments, ExecutionArguments) |
| 8 | +- arguments_typed.py: PyinfraOperation.__call__ method signature |
| 9 | +
|
| 10 | +This script ensures they stay synchronized. |
| 11 | +""" |
| 12 | + |
| 13 | +import ast |
| 14 | +import sys |
| 15 | +from os import path |
| 16 | +from typing import NamedTuple |
| 17 | + |
| 18 | + |
| 19 | +class ArgumentInfo(NamedTuple): |
| 20 | + name: str |
| 21 | + type_annotation: str |
| 22 | + has_default: bool |
| 23 | + |
| 24 | + |
| 25 | +def get_typeddict_keys(tree: ast.Module, class_names: list[str]) -> dict[str, str]: |
| 26 | + """Extract keys and their type annotations from TypedDict classes.""" |
| 27 | + keys: dict[str, str] = {} |
| 28 | + |
| 29 | + for node in ast.walk(tree): |
| 30 | + if isinstance(node, ast.ClassDef) and node.name in class_names: |
| 31 | + for item in node.body: |
| 32 | + if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name): |
| 33 | + key = item.target.id |
| 34 | + type_str = ast.unparse(item.annotation) |
| 35 | + keys[key] = type_str |
| 36 | + |
| 37 | + return keys |
| 38 | + |
| 39 | + |
| 40 | +def get_argument_meta_keys(tree: ast.Module) -> set[str]: |
| 41 | + """Extract keys from all *_argument_meta dictionaries.""" |
| 42 | + keys: set[str] = set() |
| 43 | + |
| 44 | + for node in ast.walk(tree): |
| 45 | + if isinstance(node, ast.Assign): |
| 46 | + for target in node.targets: |
| 47 | + if isinstance(target, ast.Name) and target.id.endswith("_argument_meta"): |
| 48 | + if isinstance(node.value, ast.Dict): |
| 49 | + for key in node.value.keys: |
| 50 | + if isinstance(key, ast.Constant) and isinstance(key.value, str): |
| 51 | + keys.add(key.value) |
| 52 | + |
| 53 | + return keys |
| 54 | + |
| 55 | + |
| 56 | +def get_call_parameters(tree: ast.Module, class_name: str, method_name: str) -> list[ArgumentInfo]: |
| 57 | + """Extract parameters from a class method.""" |
| 58 | + params: list[ArgumentInfo] = [] |
| 59 | + |
| 60 | + for node in ast.walk(tree): |
| 61 | + if isinstance(node, ast.ClassDef) and node.name == class_name: |
| 62 | + for item in node.body: |
| 63 | + if isinstance(item, ast.FunctionDef) and item.name == method_name: |
| 64 | + args = item.args |
| 65 | + |
| 66 | + # Count defaults - they align with the end of the args list |
| 67 | + num_defaults = len(args.defaults) |
| 68 | + num_args = len(args.args) |
| 69 | + |
| 70 | + for i, arg in enumerate(args.args): |
| 71 | + if arg.arg == "self": |
| 72 | + continue |
| 73 | + |
| 74 | + type_str = ast.unparse(arg.annotation) if arg.annotation else "" |
| 75 | + |
| 76 | + # Check if this arg has a default |
| 77 | + default_index = i - (num_args - num_defaults) |
| 78 | + has_default = default_index >= 0 |
| 79 | + |
| 80 | + params.append( |
| 81 | + ArgumentInfo( |
| 82 | + name=arg.arg, |
| 83 | + type_annotation=type_str, |
| 84 | + has_default=has_default, |
| 85 | + ) |
| 86 | + ) |
| 87 | + |
| 88 | + # Also check kwonly args |
| 89 | + for i, arg in enumerate(args.kwonlyargs): |
| 90 | + type_str = ast.unparse(arg.annotation) if arg.annotation else "" |
| 91 | + has_default = args.kw_defaults[i] is not None |
| 92 | + params.append( |
| 93 | + ArgumentInfo( |
| 94 | + name=arg.arg, |
| 95 | + type_annotation=type_str, |
| 96 | + has_default=has_default, |
| 97 | + ) |
| 98 | + ) |
| 99 | + |
| 100 | + return params |
| 101 | + |
| 102 | + |
| 103 | +def normalize_type(type_str: str) -> str: |
| 104 | + """Normalize type string for comparison (handle Optional, Union, etc.).""" |
| 105 | + # Remove whitespace |
| 106 | + type_str = type_str.replace(" ", "") |
| 107 | + |
| 108 | + # Sort Union members for consistent comparison |
| 109 | + if type_str.startswith("Union[") or type_str.startswith("Optional["): |
| 110 | + # This is a simplified normalization - just for basic comparison |
| 111 | + pass |
| 112 | + |
| 113 | + return type_str |
| 114 | + |
| 115 | + |
| 116 | +def main() -> int: |
| 117 | + this_dir = path.dirname(path.realpath(__file__)) |
| 118 | + repo_root = path.abspath(path.join(this_dir, "..")) |
| 119 | + |
| 120 | + arguments_path = path.join(repo_root, "src", "pyinfra", "api", "arguments.py") |
| 121 | + arguments_typed_path = path.join(repo_root, "src", "pyinfra", "api", "arguments_typed.py") |
| 122 | + |
| 123 | + # Parse both files |
| 124 | + with open(arguments_path, "r", encoding="utf-8") as f: |
| 125 | + arguments_tree = ast.parse(f.read()) |
| 126 | + |
| 127 | + with open(arguments_typed_path, "r", encoding="utf-8") as f: |
| 128 | + arguments_typed_tree = ast.parse(f.read()) |
| 129 | + |
| 130 | + # Extract TypedDict keys from arguments.py |
| 131 | + typeddict_classes = ["ConnectorArguments", "MetaArguments", "ExecutionArguments"] |
| 132 | + typeddict_keys = get_typeddict_keys(arguments_tree, typeddict_classes) |
| 133 | + |
| 134 | + # Extract argument meta keys (the actual source of truth for what arguments exist) |
| 135 | + meta_keys = get_argument_meta_keys(arguments_tree) |
| 136 | + |
| 137 | + # Extract PyinfraOperation.__call__ parameters from arguments_typed.py |
| 138 | + call_params = get_call_parameters(arguments_typed_tree, "PyinfraOperation", "__call__") |
| 139 | + call_param_names = {p.name for p in call_params} |
| 140 | + call_param_types = {p.name: p.type_annotation for p in call_params} |
| 141 | + |
| 142 | + errors: list[str] = [] |
| 143 | + warnings: list[str] = [] |
| 144 | + |
| 145 | + # Check that all TypedDict keys are in PyinfraOperation.__call__ |
| 146 | + for key, type_str in typeddict_keys.items(): |
| 147 | + if key not in call_param_names: |
| 148 | + errors.append( |
| 149 | + f"Argument '{key}' is in arguments.py TypedDicts but missing from " |
| 150 | + f"PyinfraOperation.__call__ in arguments_typed.py" |
| 151 | + ) |
| 152 | + elif key in call_param_types: |
| 153 | + typed_type = normalize_type(call_param_types[key]) |
| 154 | + expected_type = normalize_type(type_str) |
| 155 | + |
| 156 | + # TypedDict uses non-Optional types, but PyinfraOperation uses Optional |
| 157 | + # So we do a loose check - the base type should match |
| 158 | + # This is a simplified check that may need refinement |
| 159 | + if expected_type not in typed_type and typed_type not in expected_type: |
| 160 | + # Check if it's just an Optional wrapper difference |
| 161 | + if f"Optional[{expected_type}]" != typed_type and expected_type != typed_type: |
| 162 | + warnings.append( |
| 163 | + f"Type mismatch for '{key}': " |
| 164 | + f"arguments.py has '{type_str}', " |
| 165 | + f"arguments_typed.py has '{call_param_types[key]}'" |
| 166 | + ) |
| 167 | + |
| 168 | + # Check that all PyinfraOperation.__call__ params (except special ones) are in TypedDicts |
| 169 | + # Skip *args and **kwargs which are the P.args/P.kwargs for operation-specific args |
| 170 | + special_params = {"args", "kwargs"} |
| 171 | + for param in call_params: |
| 172 | + if param.name in special_params: |
| 173 | + continue |
| 174 | + if param.name not in typeddict_keys: |
| 175 | + errors.append( |
| 176 | + f"Parameter '{param.name}' is in PyinfraOperation.__call__ but missing from " |
| 177 | + f"TypedDicts in arguments.py" |
| 178 | + ) |
| 179 | + |
| 180 | + # Check that all meta keys are represented |
| 181 | + all_typeddict_keys = set(typeddict_keys.keys()) |
| 182 | + for key in meta_keys: |
| 183 | + if key not in all_typeddict_keys: |
| 184 | + errors.append(f"Argument '{key}' is in argument_meta dicts but missing from TypedDicts") |
| 185 | + |
| 186 | + # Report results |
| 187 | + if warnings: |
| 188 | + print("Warnings:") |
| 189 | + for warning in warnings: |
| 190 | + print(f" ⚠️ {warning}") |
| 191 | + print() |
| 192 | + |
| 193 | + if errors: |
| 194 | + print("Errors:") |
| 195 | + for error in errors: |
| 196 | + print(f" ❌ {error}") |
| 197 | + print() |
| 198 | + print(f"Found {len(errors)} error(s) and {len(warnings)} warning(s)") |
| 199 | + return 1 |
| 200 | + |
| 201 | + print("✅ arguments.py and arguments_typed.py are in sync!") |
| 202 | + if warnings: |
| 203 | + print(f" ({len(warnings)} warning(s) - type annotations may differ slightly)") |
| 204 | + return 0 |
| 205 | + |
| 206 | + |
| 207 | +if __name__ == "__main__": |
| 208 | + sys.exit(main()) |
0 commit comments