From 60e6cbd34bdf7f0aabbac41ee8deacf7626a27bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D9=8A?= Date: Mon, 2 Mar 2026 20:59:07 +0530 Subject: [PATCH] Harden SIF release readiness gaps and add regression checks --- .../intelligence/agents/agent_orchestrator.py | 6 -- .../intelligence/agents/specialized_agents.py | 58 ++++++++++-- backend/services/intelligence/sif_agents.py | 58 ++++++++++-- backend/sif_release_readiness_checks.py | 89 +++++++++++++++++++ ...DAYS_TASKS_WORKFLOW_IMPLEMENTATION_PLAN.md | 30 +++++++ 5 files changed, 219 insertions(+), 22 deletions(-) create mode 100644 backend/sif_release_readiness_checks.py diff --git a/backend/services/intelligence/agents/agent_orchestrator.py b/backend/services/intelligence/agents/agent_orchestrator.py index b00c9bce..401f4ca9 100644 --- a/backend/services/intelligence/agents/agent_orchestrator.py +++ b/backend/services/intelligence/agents/agent_orchestrator.py @@ -168,12 +168,6 @@ def _create_specialized_agents(self): self.social_agent = SocialAmplificationAgent(self.user_id, self.config.shared_llm, llm=self.llm) self.agents['social'] = self.social_agent - # Strategy Architect Agent - if enabled_by_key.get("strategy_architect", True): - from services.intelligence.txtai_service import TxtaiIntelligenceService - intel_service = TxtaiIntelligenceService(self.user_id) - self.strategy_agent = StrategyArchitectAgent(intel_service, self.user_id) - self.agents['strategy'] = self.strategy_agent # Trend Surfer Agent if enabled_by_key.get("trend_surfer", True): diff --git a/backend/services/intelligence/agents/specialized_agents.py b/backend/services/intelligence/agents/specialized_agents.py index 333799b0..59094133 100644 --- a/backend/services/intelligence/agents/specialized_agents.py +++ b/backend/services/intelligence/agents/specialized_agents.py @@ -531,15 +531,57 @@ async def verify_originality(self, text: str, competitor_index: Any) -> Dict[str logger.warning(f"[{self.__class__.__name__}] Text too short for meaningful originality check") return {"originality_score": 0.0, "reason": "Text too short"} - # STUB: Implement cross-index search against competitor content - # This would search the text against a competitor-specific index - - logger.info(f"[{self.__class__.__name__}] Originality verification stub completed") + query = text.strip() + competitor_results = [] + method = "user_index_competitor_filter" + + if competitor_index is not None and hasattr(competitor_index, "search"): + method = "competitor_index_search" + raw_results = competitor_index.search(query, limit=5) + if asyncio.iscoroutine(raw_results): + raw_results = await raw_results + competitor_results = raw_results or [] + else: + raw_results = await self.intelligence.search(query, limit=10) + for result in raw_results or []: + metadata_raw = result.get("object") + metadata = metadata_raw if isinstance(metadata_raw, dict) else {} + if not metadata and isinstance(metadata_raw, str): + try: + metadata = json.loads(metadata_raw) + except Exception: + metadata = {} + + doc_type = str((metadata or {}).get("type", "")).lower() + source = str((metadata or {}).get("source", "")).lower() + if "competitor" in doc_type or "competitor" in source: + competitor_results.append(result) + + if not competitor_results: + return { + "originality_score": 1.0, + "confidence": 0.6, + "method": method, + "notes": "No competitor overlap detected in available index" + } + + top_match = max(competitor_results, key=lambda item: float(item.get("score", 0.0))) + top_score = max(0.0, min(1.0, float(top_match.get("score", 0.0)))) + originality_score = max(0.0, round(1.0 - top_score, 4)) + confidence = round(min(1.0, 0.55 + (min(len(competitor_results), 5) * 0.07)), 3) + warning = originality_score < self.ORIGINALITY_THRESHOLD + return { - "originality_score": 0.95, # Placeholder - "confidence": 0.8, - "method": "semantic_comparison", - "notes": "Competitor index integration pending" + "originality_score": originality_score, + "confidence": confidence, + "method": method, + "warning": warning, + "threshold": self.ORIGINALITY_THRESHOLD, + "top_competitor_match": { + "id": top_match.get("id"), + "score": round(top_score, 4) + }, + "matches_evaluated": len(competitor_results) } except Exception as e: diff --git a/backend/services/intelligence/sif_agents.py b/backend/services/intelligence/sif_agents.py index 6bc697c0..d28869e5 100644 --- a/backend/services/intelligence/sif_agents.py +++ b/backend/services/intelligence/sif_agents.py @@ -544,15 +544,57 @@ async def verify_originality(self, text: str, competitor_index: Any) -> Dict[str logger.warning(f"[{self.__class__.__name__}] Text too short for meaningful originality check") return {"originality_score": 0.0, "reason": "Text too short"} - # STUB: Implement cross-index search against competitor content - # This would search the text against a competitor-specific index - - logger.info(f"[{self.__class__.__name__}] Originality verification stub completed") + query = text.strip() + competitor_results = [] + method = "user_index_competitor_filter" + + if competitor_index is not None and hasattr(competitor_index, "search"): + method = "competitor_index_search" + raw_results = competitor_index.search(query, limit=5) + if asyncio.iscoroutine(raw_results): + raw_results = await raw_results + competitor_results = raw_results or [] + else: + raw_results = await self.intelligence.search(query, limit=10) + for result in raw_results or []: + metadata_raw = result.get("object") + metadata = metadata_raw if isinstance(metadata_raw, dict) else {} + if not metadata and isinstance(metadata_raw, str): + try: + metadata = json.loads(metadata_raw) + except Exception: + metadata = {} + + doc_type = str((metadata or {}).get("type", "")).lower() + source = str((metadata or {}).get("source", "")).lower() + if "competitor" in doc_type or "competitor" in source: + competitor_results.append(result) + + if not competitor_results: + return { + "originality_score": 1.0, + "confidence": 0.6, + "method": method, + "notes": "No competitor overlap detected in available index" + } + + top_match = max(competitor_results, key=lambda item: float(item.get("score", 0.0))) + top_score = max(0.0, min(1.0, float(top_match.get("score", 0.0)))) + originality_score = max(0.0, round(1.0 - top_score, 4)) + confidence = round(min(1.0, 0.55 + (min(len(competitor_results), 5) * 0.07)), 3) + warning = originality_score < self.ORIGINALITY_THRESHOLD + return { - "originality_score": 0.95, # Placeholder - "confidence": 0.8, - "method": "semantic_comparison", - "notes": "Competitor index integration pending" + "originality_score": originality_score, + "confidence": confidence, + "method": method, + "warning": warning, + "threshold": self.ORIGINALITY_THRESHOLD, + "top_competitor_match": { + "id": top_match.get("id"), + "score": round(top_score, 4) + }, + "matches_evaluated": len(competitor_results) } except Exception as e: diff --git a/backend/sif_release_readiness_checks.py b/backend/sif_release_readiness_checks.py new file mode 100644 index 00000000..9d79e14b --- /dev/null +++ b/backend/sif_release_readiness_checks.py @@ -0,0 +1,89 @@ +import asyncio +import unittest +import sys +from pathlib import Path +from unittest.mock import patch +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.intelligence.monitoring.semantic_dashboard import RealTimeSemanticMonitor, SemanticHealthMetric +from services.today_workflow_service import _ensure_pillar_coverage, PILLAR_IDS +from services.intelligence.sif_agents import ContentGuardianAgent as SifGuardian +from services.intelligence.agents.specialized_agents import ContentGuardianAgent as SpecializedGuardian + + +class _FakeIntelligence: + def __init__(self, results=None): + self._results = results or [] + + async def search(self, query: str, limit: int = 10): + return self._results + + +class _FakeCompetitorIndex: + async def search(self, query: str, limit: int = 5): + return [ + {"id": "comp-1", "score": 0.82}, + {"id": "comp-2", "score": 0.65}, + ] + + +class SIFReleaseReadinessTests(unittest.IsolatedAsyncioTestCase): + def test_single_strategy_architect_init_block(self): + source = Path("backend/services/intelligence/agents/agent_orchestrator.py").read_text() + self.assertEqual(source.count('if enabled_by_key.get("strategy_architect", True):'), 1) + + async def test_semantic_health_returns_canonical_metric(self): + monitor = RealTimeSemanticMonitor.__new__(RealTimeSemanticMonitor) + monitor.user_id = "u1" + metric_list = [ + SemanticHealthMetric("semantic_diversity", 0.8, 0.6, "healthy", "t", "d", []), + SemanticHealthMetric("authority_score", 0.3, 0.4, "critical", "t", "d", ["Improve authority"]), + ] + async def _fake_metrics(): + return metric_list + monitor._check_semantic_health = _fake_metrics + + result = await RealTimeSemanticMonitor.check_semantic_health(monitor) + self.assertIsInstance(result, SemanticHealthMetric) + self.assertEqual(result.metric_name, "semantic_health") + self.assertEqual(result.status, "critical") + + async def test_verify_originality_uses_real_scores_sif_guardian(self): + agent = SifGuardian.__new__(SifGuardian) + agent.ORIGINALITY_THRESHOLD = 0.75 + agent.intelligence = _FakeIntelligence() + agent._log_agent_operation = lambda *args, **kwargs: None + + result = await SifGuardian.verify_originality(agent, "This is sufficiently long text for originality analysis.", _FakeCompetitorIndex()) + self.assertIn("originality_score", result) + self.assertLess(result["originality_score"], 1.0) + self.assertIn("warning", result) + self.assertEqual(result["method"], "competitor_index_search") + + async def test_verify_originality_uses_real_scores_specialized_guardian(self): + agent = SpecializedGuardian.__new__(SpecializedGuardian) + agent.ORIGINALITY_THRESHOLD = 0.75 + agent.intelligence = _FakeIntelligence() + agent._log_agent_operation = lambda *args, **kwargs: None + + result = await SpecializedGuardian.verify_originality(agent, "This is sufficiently long text for originality analysis.", _FakeCompetitorIndex()) + self.assertIn("originality_score", result) + self.assertLess(result["originality_score"], 1.0) + self.assertIn("warning", result) + self.assertEqual(result["method"], "competitor_index_search") + + def test_pillar_coverage_guardrail_backfills_missing(self): + tasks = [{"pillarId": "plan", "title": "Plan", "description": "d", "priority": "high", "estimatedTime": 10, "actionType": "navigate", "enabled": True}] + grounding = {"workflow_config": {"enforce_pillar_coverage": True}} + + with patch("services.today_workflow_service._build_single_task_for_missing_pillar", return_value=None): + covered = _ensure_pillar_coverage(tasks, "u1", "2026-01-01", grounding) + + pillars = {t["pillarId"] for t in covered} + self.assertEqual(pillars, set(PILLAR_IDS)) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/SIF/TODAYS_TASKS_WORKFLOW_IMPLEMENTATION_PLAN.md b/docs/SIF/TODAYS_TASKS_WORKFLOW_IMPLEMENTATION_PLAN.md index 812124f8..bf3fd55d 100644 --- a/docs/SIF/TODAYS_TASKS_WORKFLOW_IMPLEMENTATION_PLAN.md +++ b/docs/SIF/TODAYS_TASKS_WORKFLOW_IMPLEMENTATION_PLAN.md @@ -271,3 +271,33 @@ The Today's Tasks Workflow System is designed to transform ALwrity's complex dig --- *This document serves as the foundation for implementing the Today's Tasks Workflow System. It should be reviewed and updated regularly as the project progresses and new insights are gained.* + +## ✅ **Production Release Gates (SIF Today Workflow)** + +Before enabling broad production rollout, verify the following release gates: + +1. **Committee completeness** + - `StrategyArchitectAgent` is initialized once and included in committee polling. + - Daily workflow generation includes all major agent lanes (`strategy`, `content`, `seo`, `social`, `competitor`). + +2. **Semantic intelligence quality gates** + - `find_semantic_gaps` returns evidence-backed topic gaps (no placeholder outputs). + - `verify_originality` uses real competitor similarity evidence (no synthetic placeholder score). + +3. **Response contract gates** + - Semantic health API returns the canonical `SemanticHealthMetric` structure in success and fallback paths. + - Workflow task payloads always pass pillar coverage guardrails unless explicitly disabled by config. + +4. **Observability gates** + - Error paths emit structured logs with `user_id` and operation context. + - Semantic filter degraded-path counters are incremented and logged. + +5. **Test and validation gates** + - Automated tests cover: + - single strategy agent initialization path, + - semantic health aggregation contract, + - originality score computation path, + - pillar coverage backfill behavior. + - Static compile check passes on modified backend modules. + +**Release recommendation**: proceed only when all gates pass in CI and staging smoke tests.