smith/.env_template at main · IBM/smith · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
# Project-specific configuration
# Example: BASE_URL=/Users/smith_user/.bob/skills/smith/
BASE_URL=base url of your skill folder
DATA_DIR=assets/opa/
POLICY_DIR=assets/
POLICY_PATH=policy.rego

# Model configuration
MODEL_SONNET=aws/claude-sonnet-4-6
TEMP=0.2
TOP_P=0.9
OPENAI_API_KEY=your api key
OPENAI_BASE_URL=your base url

# Target agent configuration
AGENT_URL=http://localhost:9000

# INFERENCE_* configure the example agent's own LLM client (examples/*), not Smith itself.
# Option A: Ollama (local)
# INFERENCE_MODEL="qwen3.5:latest"
# INFERENCE_BASE_URL="http://localhost:11434/v1"
# INFERENCE_API_KEY="ollama"
# OLLAMA_BASE_URL is read by promptfoo for red-team generation (native ollama API, no /v1 suffix).
# Leave unset to use the promptfoo default http://localhost:11434
OLLAMA_BASE_URL="http://localhost:11434"
# Option B: RITS (IBM internal, requires VPN)
# INFERENCE_MODEL=your rits model
# INFERENCE_BASE_URL=your rits base url
# INFERENCE_API_KEY=your rits api key
# Option C: online model over the same gateway Smith itself uses. Keep the model
# distinct from MODEL_SONNET: that one drives Smith's pipelines, this one drives
# the target agent. Set TRANSLATION_CONCURRENCY below to send requests in parallel.
INFERENCE_MODEL=aws/claude-haiku-4-5
INFERENCE_BASE_URL=${OPENAI_BASE_URL}
INFERENCE_API_KEY=${OPENAI_API_KEY}

TARGET_AGENT_PATH=examples/hr-agent/
GUIDANCE_FILE=examples/hr-agent/smith/guidance.txt
SYSTEM_VAR_FILE=examples/hr-agent/smith/system_vars.json
PROMPTFOO_CONFIG_FILE=examples/hr-agent/smith/promptfooconfig.yaml
PROMPTFOO_OUTPUT_FILE=examples/hr-agent/smith/redteam.yaml

PROMPTFOO_CONFIG_TEMPLATE=references/promptfoo_config_template.yaml

## MCP settings: for get_mcp_parameters when generating the policy
MCP_TRANSPORT=http
MCP_URL=http://localhost:9000/tool_definitions

# Agent-side only: where an example agent POSTs its own tool calls, when it
# talks to an MCP server separate from the endpoint Smith reads definitions
# from (e.g. examples/hr-agent, whose MCP server.py listens on :9100).
# MCP_CALL_URL=http://localhost:9100/mcp

# example for stdio MCP
# MCP_TRANSPORT=stdio
# MCP_COMMAND=python
# MCP_ARGS=server.py
# MCP_CWD=examples/call-for-papers-mcp

# example for nodejs MCP
# MCP_TRANSPORT=stdio
# MCP_COMMAND=node
# MCP_ARGS=dist/index.js
# MCP_CWD=examples/context7-mcp


# Policy testing configuration
BAD_COMMAND_PATH=references/test_cases/allow
BENIGN_COMMAND_PATH=references/test_cases/disallow
TEST_CASE_PATH=references/test_cases/
# Where the policy_testing harness writes the scorecard + failure list
# (relative to BASE_URL). The harness itself ships inside the smith package.
TEST_OUTPUT_DIR=references/scorecard/
TEST_PATH=tests/integration/
TEST_RESULT_PATH=scorecard_summary.txt
TEST_FAILURES_PATH=score_test_failures.txt
CROSS_VALIDATE_OUTPUT=references/cross_validate_report.json

# Session configuration (written by the explorer UI, read by the CLI)
SESSION_CONFIG_FILE=references/session_config.json

# Test case generation configuration
TEST_CASE_TEMPLATE=references/test_case_template.json
TEST_GENERATION_PATH=src/smith/test_generation/
DECOMP_FILE=references/decomp_file.json
FLATTEN_FILE=references/decomp_flatten_file.json
ATTACK_FILE_CSV=ares/assets/safety_behaviors_text_subset.csv
VARS_FILE=references/vars_file.json
ATTACK_FILE=references/decomp_attack_file.json
ATTACK_FILE_PROMPT=references/decomp_attack_file_promptfoo.json
CASE_FILE=references/test_cases.json
FINAL_TEST_CASES=references/test_cases/
GREY_GUIDANCE_FILE=references/grey_guidance.json

# Incremental test generation (smith --flag test_generation --mode update)
# Snapshot of the flattened guidance the last run generated from.
GUIDANCE_SNAPSHOT_FILE=references/guidance_snapshot.txt
# Snapshot of the raw guidance file the last run used.
GUIDANCE_RAW_SNAPSHOT_FILE=references/guidance_raw_snapshot.txt
# Maps each guidance line -> the test case files generated from it, so an update
# run can remove exactly the cases whose guidance changed.
GUIDANCE_MAP_FILE=references/guidance_case_map.json

# Abstract bypass cases synthesized from the policy analysis.
BYPASS_CASE_FILE=references/bypass_cases.json
# Directory for the structured bypass vector report (JSON + Markdown).
BYPASS_REPORT_DIR=references/bypass/

BATCH_PROCESSING=true
BATCH_SIZE=10
CASE_GENERATION_BATCH_SIZE=5
# Batches dispatched in parallel per stage during test_generation; set it to 1 (or -1) to send one batch at a time.
GENERATION_CONCURRENCY=4
# Concurrent /extract_tool_call requests during test_case_translation.
# -1 (or 1) sends one at a time -- use this for a local model, which is
# compute-bound and gains nothing from concurrency. Raise it for an online model.
TRANSLATION_CONCURRENCY=8

## Red teaming tool configuration
# Comma-separated list of attack tools to run during test generation.
# Valid values: ares, promptfoo, ares,promptfoo, none
ATTACK_TOOLS=promptfoo
ARES_HOME=path to your ares project directory
# Disable promptfoo remote connection
PROMPTFOO_DISABLE_TELEMETRY=1
PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION=true
PROMPTFOO_DISABLE_SHARING=true


# Test case evaluation configuration
CLASSIFIED_PROMPTFOO_FILE=references/decomp_attack_file_promptfoo_classified.json
CLASSIFY_TOP_N=3
### Label-validation tiers (all optional; defaults shown)
TIER2_HIGH_THRESHOLD=0.70
TIER2_LOW_THRESHOLD=0.35
### Cap on Tier-3 LLM judge calls (blank = unlimited)
MAX_LLM_CALLS=


# Policy refinement configuration

## Patching agent configuration
BACKUP_NAME='tmp'
### for red feedback update
CLUSTER_RESULTS=cluster_results.txt
### DBSCAN clustering parameters for grouping similar test case failures
### EPS: maximum cosine distance between two samples to be in the same cluster (smaller = tighter clusters)
CLUSTER_EPS=0.3
### MIN_SAMPLES: minimum number of samples required to form a cluster (samples below this threshold become noise)
CLUSTER_MIN_SAMPLES=2

## Linting agent configuration
GRAPH_PATH=ast.dot
OPA_AST_PATH=ast.json
GRAPH_SUGGESTION_PATH=graph_suggestion.txt
### for duplication feedback update
MODIFIED_POLICY_DEDUPLICATE=deduplicate_policy.rego
### for regal feedback update
REGAL_RESULT_OUTPUT=regal_result_output.txt
MODIFIED_POLICY_REGAL=regal_policy.rego
REGAL_SUGGESTION_PATH=regal_suggestion.txt