Improve workflow reliability, issue dedup, and update AI model (#102)

mikejamescalvert · claude · web-flow · commit 8989f217999d · 2026-03-17T18:42:28.000-04:00
- Fix cache persistence gaps: add cache/save steps to daily-screener and performance-report workflows (data was being lost between runs) - Add failure alert issues to all 5 agent workflows: on failure, creates or comments on an existing agent-failure issue with run link - Add issues:write permission to trading-agent workflow for failure alerts - Improve portfolio review issue deduplication: add title-similarity matching (keyword overlap) on top of existing category-based dedup to prevent repeated themes like "add crypto" across different categories - Add stale issue cleanup to watchlist review agent (auto-close >7 days) - Update default AI model from claude-sonnet-4-20250514 to claude-sonnet-4-6-20250620 across all workflows and config https://claude.ai/code/session_016RNRhxDH2k1GVjnaRmakmZ Co-authored-by: Claude <noreply@anthropic.com>
diff --git a/.github/workflows/daily-screener.yml b/.github/workflows/daily-screener.yml
@@ -60,9 +60,39 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: python -m financial_agent.screener_main
 
+      - name: Save persistent data
+        if: always()
+        uses: actions/cache/save@v5
+        with:
+          path: .data
+          key: agent-data-${{ github.run_id }}
+
       - name: Post summary
         if: always()
         run: |
           echo "## Daily Screener" >> $GITHUB_STEP_SUMMARY
           echo "- **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY
           echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Create failure issue
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          EXISTING=$(gh issue list --label "agent-failure" --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "Daily Screener failed at $(date -u '+%Y-%m-%d %H:%M UTC'). Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          else
+            gh label create agent-failure --color D93F0B --force 2>/dev/null || true
+            gh issue create \
+              --title "Daily Screener workflow failure" \
+              --label "agent-failure" \
+              --body "## Daily Screener Failed
+
+          **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')
+          **Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+          Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+
+          _This issue was automatically created by the CI failure alert._"
+          fi
diff --git a/.github/workflows/performance-report.yml b/.github/workflows/performance-report.yml
@@ -58,9 +58,39 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: python -m financial_agent.performance_main
 
+      - name: Save persistent data
+        if: always()
+        uses: actions/cache/save@v5
+        with:
+          path: .data
+          key: agent-data-${{ github.run_id }}
+
       - name: Post summary
         if: always()
         run: |
           echo "## Performance Report" >> $GITHUB_STEP_SUMMARY
           echo "- **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY
           echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Create failure issue
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          EXISTING=$(gh issue list --label "agent-failure" --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "Performance Report failed at $(date -u '+%Y-%m-%d %H:%M UTC'). Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          else
+            gh label create agent-failure --color D93F0B --force 2>/dev/null || true
+            gh issue create \
+              --title "Performance Report workflow failure" \
+              --label "agent-failure" \
+              --body "## Performance Report Failed
+
+          **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')
+          **Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+          Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+
+          _This issue was automatically created by the CI failure alert._"
+          fi
diff --git a/.github/workflows/portfolio-review.yml b/.github/workflows/portfolio-review.yml
@@ -57,7 +57,7 @@ jobs:
 
           # Configuration (reuses trading variables)
           ALPACA_BASE_URL: ${{ vars.ALPACA_BASE_URL || 'https://paper-api.alpaca.markets' }}
-          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-20250514' }}
+          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-6-20250620' }}
           TRADING_MAX_POSITION_PCT: ${{ vars.TRADING_MAX_POSITION_PCT || '0.10' }}
           TRADING_MAX_DAILY_TRADES: ${{ vars.TRADING_MAX_DAILY_TRADES || '10' }}
           TRADING_STOP_LOSS_PCT: ${{ vars.TRADING_STOP_LOSS_PCT || '0.05' }}
@@ -84,3 +84,26 @@ jobs:
           echo "## Portfolio Review" >> $GITHUB_STEP_SUMMARY
           echo "- **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY
           echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Create failure issue
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          EXISTING=$(gh issue list --label "agent-failure" --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "Portfolio Review failed at $(date -u '+%Y-%m-%d %H:%M UTC'). Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          else
+            gh label create agent-failure --color D93F0B --force 2>/dev/null || true
+            gh issue create \
+              --title "Portfolio Review workflow failure" \
+              --label "agent-failure" \
+              --body "## Portfolio Review Failed
+
+          **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')
+          **Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+          Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+
+          _This issue was automatically created by the CI failure alert._"
+          fi
diff --git a/.github/workflows/trading-agent.yml b/.github/workflows/trading-agent.yml
@@ -35,6 +35,7 @@ concurrency:
 
 permissions:
   contents: read
+  issues: write
 
 jobs:
   trade:
@@ -78,7 +79,7 @@ jobs:
 
           # Configuration (from GitHub Variables)
           ALPACA_BASE_URL: ${{ vars.ALPACA_BASE_URL || 'https://paper-api.alpaca.markets' }}
-          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-20250514' }}
+          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-6-20250620' }}
           TRADING_MAX_POSITION_PCT: ${{ vars.TRADING_MAX_POSITION_PCT || '0.10' }}
           TRADING_MAX_DAILY_TRADES: ${{ vars.TRADING_MAX_DAILY_TRADES || '10' }}
           TRADING_STOP_LOSS_PCT: ${{ vars.TRADING_STOP_LOSS_PCT || '0.05' }}
@@ -122,3 +123,27 @@ jobs:
           echo "- **Dry Run:** ${{ inputs.dry_run || vars.TRADING_DRY_RUN || 'false' }}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "The agent crashed before completing. Check the logs above for details." >> $GITHUB_STEP_SUMMARY
+
+      - name: Create failure issue
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          EXISTING=$(gh issue list --label "agent-failure" --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "Trading Agent failed again at $(date -u '+%Y-%m-%d %H:%M UTC'). Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          else
+            gh label create agent-failure --color D93F0B --force 2>/dev/null || true
+            gh issue create \
+              --title "Trading Agent workflow failure" \
+              --label "agent-failure" \
+              --body "## Trading Agent Failed
+
+          **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')
+          **Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          **Strategy:** ${{ inputs.strategy || vars.TRADING_STRATEGY || 'balanced' }}
+
+          Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+
+          _This issue was automatically created by the CI failure alert._"
+          fi
diff --git a/.github/workflows/watchlist-review.yml b/.github/workflows/watchlist-review.yml
@@ -58,7 +58,7 @@ jobs:
 
           # Configuration
           ALPACA_BASE_URL: ${{ vars.ALPACA_BASE_URL || 'https://paper-api.alpaca.markets' }}
-          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-20250514' }}
+          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL || 'claude-sonnet-4-6-20250620' }}
           TRADING_WATCHLIST: ${{ vars.TRADING_WATCHLIST || 'AAPL,MSFT,GOOGL,AMZN,NVDA,META,TSLA,JPM,V,JNJ' }}
           TRADING_CRYPTO_WATCHLIST: ${{ vars.TRADING_CRYPTO_WATCHLIST || 'BTC/USD,ETH/USD,SOL/USD' }}
           TRADING_STOCK_UNIVERSE: ${{ vars.TRADING_STOCK_UNIVERSE || '' }}
@@ -88,3 +88,26 @@ jobs:
           echo "## Watchlist Review" >> $GITHUB_STEP_SUMMARY
           echo "- **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY
           echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Create failure issue
+        if: failure()
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+        run: |
+          EXISTING=$(gh issue list --label "agent-failure" --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "Watchlist Review failed at $(date -u '+%Y-%m-%d %H:%M UTC'). Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          else
+            gh label create agent-failure --color D93F0B --force 2>/dev/null || true
+            gh issue create \
+              --title "Watchlist Review workflow failure" \
+              --label "agent-failure" \
+              --body "## Watchlist Review Failed
+
+          **Time:** $(date -u '+%Y-%m-%d %H:%M UTC')
+          **Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+          Check the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+
+          _This issue was automatically created by the CI failure alert._"
+          fi
diff --git a/src/financial_agent/config.py b/src/financial_agent/config.py
@@ -34,7 +34,7 @@ class AIConfig(BaseSettings):
 
     api_key: str = Field(description="Anthropic API key (GitHub Secret: ANTHROPIC_API_KEY)")
     model: str = Field(
-        default="claude-sonnet-4-20250514",
+        default="claude-sonnet-4-6-20250620",
         description="Claude model to use (GitHub Variable: ANTHROPIC_MODEL)",
     )
     max_tokens: int = Field(
diff --git a/src/financial_agent/review_main.py b/src/financial_agent/review_main.py
@@ -36,11 +36,10 @@ def _run_gh_command(cmd: list[str], timeout: int = 30) -> tuple[bool, str]:
         return False, ""
 
 
-def _get_open_review_categories() -> set[str]:
-    """Get the set of categories that already have open portfolio-review issues.
+def _get_open_review_issues() -> tuple[set[str], list[str]]:
+    """Get categories and titles of open portfolio-review issues for deduplication.
 
-    Returns category labels (e.g. 'risk', 'strategy', 'config') that have at least
-    one open issue, so we can skip creating duplicates.
+    Returns a tuple of (category_set, title_list) from open portfolio-review issues.
     """
     log = structlog.get_logger()
     success, output = _run_gh_command(
@@ -55,28 +54,80 @@ def _get_open_review_categories() -> set[str]:
             "--limit",
             "30",
             "--json",
-            "labels",
+            "labels,title",
         ],
     )
     if not success or not output:
-        return set()
+        return set(), []
 
     try:
         issues = json.loads(output)
     except json.JSONDecodeError:
-        return set()
+        return set(), []
 
     # Extract category labels from existing open issues
     category_labels = {"risk", "performance", "strategy", "config", "watchlist"}
-    existing: set[str] = set()
+    existing_categories: set[str] = set()
+    existing_titles: list[str] = []
     for issue in issues:
+        existing_titles.append(issue.get("title", "").lower())
         for label in issue.get("labels", []):
             name = label.get("name", "") if isinstance(label, dict) else str(label)
             if name in category_labels:
-                existing.add(name)
+                existing_categories.add(name)
 
-    log.info("existing_review_categories", categories=sorted(existing), open_issues=len(issues))
-    return existing
+    log.info(
+        "existing_review_issues",
+        categories=sorted(existing_categories),
+        open_issues=len(issues),
+    )
+    return existing_categories, existing_titles
+
+
+def _is_duplicate_title(new_title: str, existing_titles: list[str]) -> bool:
+    """Check if a new issue title is too similar to an existing open issue.
+
+    Uses keyword overlap: if 50%+ of significant words in the new title appear
+    in an existing title, it's considered a duplicate.
+    """
+    stop_words = {
+        "a",
+        "an",
+        "the",
+        "to",
+        "for",
+        "of",
+        "in",
+        "on",
+        "and",
+        "or",
+        "with",
+        "from",
+        "by",
+        "at",
+        "is",
+        "it",
+        "as",
+        "be",
+        "this",
+        "that",
+        "into",
+        "current",
+        "based",
+    }
+    new_words = {w for w in new_title.lower().split() if w not in stop_words and len(w) > 2}
+    if not new_words:
+        return False
+
+    for existing in existing_titles:
+        existing_words = {w for w in existing.lower().split() if w not in stop_words and len(w) > 2}
+        if not existing_words:
+            continue
+        overlap = new_words & existing_words
+        if len(overlap) >= len(new_words) * 0.5:
+            return True
+
+    return False
 
 
 def _close_stale_review_issues() -> int:
@@ -253,8 +304,8 @@ def main() -> None:
     # Close stale issues before creating new ones
     _close_stale_review_issues()
 
-    # Check which categories already have open issues to avoid duplicates
-    existing_categories = _get_open_review_categories()
+    # Check which categories/titles already have open issues to avoid duplicates
+    existing_categories, existing_titles = _get_open_review_issues()
 
     # Ensure labels exist
     all_labels: set[str] = {"portfolio-review"}
@@ -278,7 +329,13 @@ def main() -> None:
 
         # Skip if an open issue already covers this category
         if category and category in existing_categories:
-            log.info("issue_skipped_duplicate", title=title, category=category)
+            log.info("issue_skipped_duplicate_category", title=title, category=category)
+            skipped += 1
+            continue
+
+        # Skip if a similar title already exists (prevents repeated themes across categories)
+        if _is_duplicate_title(title, existing_titles):
+            log.info("issue_skipped_duplicate_title", title=title)
             skipped += 1
             continue
 
@@ -315,9 +372,10 @@ def main() -> None:
 
         if _create_github_issue(title, issue_body, labels):
             issues_created += 1
-            # Track newly created category to avoid duplicates within same run
+            # Track newly created category/title to avoid duplicates within same run
             if category:
                 existing_categories.add(category)
+            existing_titles.append(title.lower())
 
     log.info(
         "review_agent_complete",
diff --git a/src/financial_agent/watchlist_main.py b/src/financial_agent/watchlist_main.py

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ class AIConfig(BaseSettings):`
`34`	`34`
`35`	`35`	`api_key: str = Field(description="Anthropic API key (GitHub Secret: ANTHROPIC_API_KEY)")`
`36`	`36`	`model: str = Field(`
`37`		`- default="claude-sonnet-4-20250514",`
	`37`	`+ default="claude-sonnet-4-6-20250620",`
`38`	`38`	`description="Claude model to use (GitHub Variable: ANTHROPIC_MODEL)",`
`39`	`39`	`)`
`40`	`40`	`max_tokens: int = Field(`