From aa77c2ec20974074db284e9171e6846091bc6bb8 Mon Sep 17 00:00:00 2001 From: Joseph Pollack Date: Mon, 8 Dec 2025 13:48:24 +0100 Subject: [PATCH 1/2] attempts to re add openalex and other tools with better search queries --- docs/troubleshooting/serper-403-error.md | 204 ++++++++++++++++++ .../serper-free-tier-optimization.md | 143 ++++++++++++ src/app.py | 13 ++ src/tools/fallback_web_search.py | 117 ++++++++++ src/tools/query_utils.py | 57 +++++ src/tools/rate_limiter.py | 23 +- src/tools/serper_web_search.py | 42 +++- src/tools/vendored/serper_client.py | 41 +++- src/tools/web_search_factory.py | 14 +- 9 files changed, 634 insertions(+), 20 deletions(-) create mode 100644 docs/troubleshooting/serper-403-error.md create mode 100644 docs/troubleshooting/serper-free-tier-optimization.md create mode 100644 src/tools/fallback_web_search.py diff --git a/docs/troubleshooting/serper-403-error.md b/docs/troubleshooting/serper-403-error.md new file mode 100644 index 00000000..005080b9 --- /dev/null +++ b/docs/troubleshooting/serper-403-error.md @@ -0,0 +1,204 @@ +# SERPER API 403 Forbidden Error - Troubleshooting Guide + +## What is the Error? + +The error you're seeing is: +``` +error="403, message='Forbidden', url='https://google.serper.dev/search'" +Serper API request failed: 403, message='Forbidden' +``` + +A **403 Forbidden** HTTP status code from the Serper API means your request was rejected due to authentication/authorization issues. + +## Root Causes + +### 1. **Invalid or Missing API Key** (Most Common) +- The `SERPER_API_KEY` environment variable is either: + - Not set + - Set to an invalid/expired key + - Contains extra whitespace or formatting issues + - Typo in the key value + +### 2. **API Key Permissions** +- The API key doesn't have permission to access the Serper API +- The key might be for a different Serper service/endpoint + +### 3. **Account/Billing Issues** +- Your Serper account might be: + - Suspended + - Over quota/limit + - Has billing issues (payment failed) + - Account expired + +### 4. **API Key Format Issues** +- The key might be malformed +- Missing characters +- Wrong key type (e.g., using a test key in production) + +## How to Diagnose + +### Step 1: Verify API Key is Set + +Check if the environment variable is set: + +**Linux/Mac:** +```bash +echo $SERPER_API_KEY +``` + +**Windows (PowerShell):** +```powershell +$env:SERPER_API_KEY +``` + +**Windows (CMD):** +```cmd +echo %SERPER_API_KEY% +``` + +### Step 2: Test the API Key Directly + +Test your Serper API key using curl: + +```bash +curl -X POST https://google.serper.dev/search \ + -H "X-API-KEY: YOUR_API_KEY_HERE" \ + -H "Content-Type: application/json" \ + -d '{"q": "test query"}' +``` + +If you get a 403, the key is invalid. If you get 200, the key works. + +### Step 3: Check Serper Dashboard + +1. Go to [Serper.dev Dashboard](https://serper.dev/dashboard) +2. Log in to your account +3. Check: + - API key status + - Usage/quota limits + - Billing status + - Account status + +## Solutions + +### Solution 1: Get a Valid Serper API Key + +1. **Sign up for Serper:** + - Visit [serper.dev](https://serper.dev) + - Create an account or log in + +2. **Get your API key:** + - Go to Dashboard → API Keys + - Copy your API key + +3. **Set the environment variable:** + + **Linux/Mac:** + ```bash + export SERPER_API_KEY="your_api_key_here" + ``` + + **Windows (PowerShell):** + ```powershell + $env:SERPER_API_KEY = "your_api_key_here" + ``` + + **Windows (CMD):** + ```cmd + set SERPER_API_KEY=your_api_key_here + ``` + + **For Hugging Face Spaces:** + - Go to your Space settings + - Add `SERPER_API_KEY` as a secret/environment variable + +4. **Verify it's set correctly:** + ```bash + # Check for whitespace issues + echo "|$SERPER_API_KEY|" # Should show key between pipes with no extra spaces + ``` + +### Solution 2: Use a Different Search Provider (Temporary Fix) + +If you can't fix the Serper API key immediately, switch to DuckDuckGo (free, no API key required): + +**Set environment variable:** +```bash +export WEB_SEARCH_PROVIDER="duckduckgo" +``` + +Or in your `.env` file: +```env +WEB_SEARCH_PROVIDER=duckduckgo +``` + +**Note:** DuckDuckGo provides lower quality results (snippets only, no full content scraping) but will work without an API key. + +### Solution 3: Check Account Status + +1. Log into [Serper Dashboard](https://serper.dev/dashboard) +2. Verify: + - Account is active + - No billing issues + - Quota not exceeded + - API key is not revoked + +### Solution 4: Regenerate API Key + +If your key might be compromised or invalid: + +1. Go to Serper Dashboard +2. Revoke the old key +3. Generate a new API key +4. Update your environment variable + +## Code Improvements (Optional) + +The current code doesn't handle 403 errors specifically. Here's what could be improved: + +### Current Behavior +- 403 errors are treated as generic `SearchError` +- System retries 3 times (wasteful - 403 won't fix itself) +- No automatic fallback to other providers + +### Recommended Improvements + +1. **Detect 403 specifically** and treat as configuration error (not transient) +2. **Disable Serper** after 403 and fall back to DuckDuckGo +3. **Log clearer error messages** with troubleshooting hints + +## Quick Fix for Your Current Issue + +**Immediate workaround:** + +1. **Remove or comment out SERPER_API_KEY:** + ```bash + unset SERPER_API_KEY + ``` + +2. **Set provider to DuckDuckGo:** + ```bash + export WEB_SEARCH_PROVIDER="duckduckgo" + ``` + +3. **Restart your application** + +This will use DuckDuckGo instead of Serper, allowing your research to continue (though with lower quality results). + +## Prevention + +1. **Validate API keys at startup** - Check if key works before using it +2. **Use environment variable validation** - Ensure key format is correct +3. **Monitor API usage** - Set up alerts for quota limits +4. **Have fallback providers** - Always have DuckDuckGo as backup + +## Summary + +- **403 Forbidden** = Invalid/missing API key or account issues +- **Fix:** Get valid Serper API key from [serper.dev](https://serper.dev) +- **Workaround:** Use `WEB_SEARCH_PROVIDER=duckduckgo` for free search +- **Test:** Use curl to verify your API key works +- **Check:** Serper dashboard for account/billing status + + + diff --git a/docs/troubleshooting/serper-free-tier-optimization.md b/docs/troubleshooting/serper-free-tier-optimization.md new file mode 100644 index 00000000..2ff20963 --- /dev/null +++ b/docs/troubleshooting/serper-free-tier-optimization.md @@ -0,0 +1,143 @@ +# Serper Free Tier Optimization + +## Problem + +Free Serper API keys have limited credits: +- **2,500 credits** (one-time, expire after 6 months) +- **100 requests/second** rate limit +- Each successful API query consumes 1 credit (or 2 if requesting >10 results) + +When credits are exhausted, Serper returns **403 Forbidden** errors. The application was treating all 403 errors as invalid keys and failing immediately, without retries. + +## Solution + +We've implemented several optimizations to better handle free tier quotas: + +### 1. Proper Rate Limiting + +**Before:** 10 requests/second (below free tier limit but not optimized) + +**After:** 90 requests/second (safely under 100/second free tier limit) + +```python +# src/tools/rate_limiter.py +def get_serper_limiter(api_key: str | None = None) -> RateLimiter: + # Free tier: 90/second (safely under 100/second limit) + return RateLimiterFactory.get("serper", "90/second") +``` + +This ensures: +- Stays safely under 100 requests/second free tier limit +- Allows high throughput when needed +- Credits are the limiting factor (2,500 total), not rate + +### 2. Jitter for Request Spreading + +Added random jitter (0-1 second) after acquiring rate limit permission: + +```python +# src/tools/rate_limiter.py +async def acquire(self, wait: bool = True, jitter: bool = False) -> bool: + if self._limiter.hit(self._rate_limit, self._identity): + if jitter: + # Add 0-1 second random jitter + jitter_seconds = random.uniform(0, 1.0) + await asyncio.sleep(jitter_seconds) + return True +``` + +**Benefits:** +- Prevents "thundering herd" - multiple parallel requests hitting at once +- Spreads load slightly to avoid bursts +- Minimal delay (max 1 second) + +### 3. Retry Logic for Credit Exhaustion + +**Before:** 403 errors → `ConfigurationError` → No retries + +**After:** 403 errors → `RateLimitError` → Retry with exponential backoff + +```python +# src/tools/vendored/serper_client.py +if response.status == 403: + # Treat as credit exhaustion (retryable) not invalid key + raise RateLimitError("Serper API credits may be exhausted...") +``` + +```python +# src/tools/serper_web_search.py +@retry( + stop=stop_after_attempt(5), # 5 retries + wait=wait_random_exponential( + multiplier=2, min=5, max=120, exp_base=2 + ), # 5s to 120s backoff with jitter +) +``` + +**Retry Schedule:** +- Attempt 1: Immediate +- Attempt 2: Wait 5-10s (with jitter) +- Attempt 3: Wait 10-20s (with jitter) +- Attempt 4: Wait 20-40s (with jitter) +- Attempt 5: Wait 40-80s (with jitter) + +### 4. Better Error Messages + +The system now distinguishes between: +- **Invalid API key** (ConfigurationError) - No retries, immediate failure +- **Quota exhaustion** (RateLimitError) - Retries with backoff + +## Usage + +The optimizations are **automatic** - no configuration needed. The system will: + +1. **Rate limit** to 1 request per 60 seconds +2. **Add jitter** to spread requests +3. **Retry on 403** with exponential backoff +4. **Log clearly** when quota is exhausted + +## Monitoring + +Watch for these log messages: + +``` +Serper API returned 403 Forbidden +May be quota exhaustion (free tier) or invalid key +Retrying with backoff... +``` + +If you see repeated 403 errors even after retries, your quota is likely exhausted for the day/month. + +## Recommendations for Free Tier + +1. **Monitor usage** at [serper.dev/dashboard](https://serper.dev/dashboard) +2. **Use DuckDuckGo fallback** when quota exhausted: + ```bash + export WEB_SEARCH_PROVIDER=duckduckgo + ``` +3. **Consider paid tier** if you need more requests +4. **Batch requests** - the rate limiter will automatically space them out + +## Configuration + +To adjust rate limiting (not recommended for free tier): + +```python +# In src/tools/rate_limiter.py +# Change from: +return RateLimiterFactory.get("serper", "1/60second") + +# To (for paid tier): +return RateLimiterFactory.get("serper", "10/second") +``` + +## Summary + +✅ **Proper rate limiting** (90 req/s, under 100/s limit) +✅ **Jitter** to spread requests (0-1s) +✅ **Retry logic** for credit exhaustion +✅ **Better error handling** +✅ **Automatic** - no config needed + +**Note:** Free tier provides 2,500 credits (one-time, expire after 6 months). Monitor usage at [serper.dev/dashboard](https://serper.dev/dashboard). Once credits are exhausted, you'll need to upgrade to a paid plan or use DuckDuckGo fallback. + diff --git a/src/app.py b/src/app.py index e700eaec..68007d89 100644 --- a/src/app.py +++ b/src/app.py @@ -81,12 +81,25 @@ def configure_orchestrator( Returns: Tuple of (orchestrator, backend_info_string) """ + from src.tools.clinicaltrials import ClinicalTrialsTool + from src.tools.europepmc import EuropePMCTool + from src.tools.pubmed import PubMedTool from src.tools.search_handler import SearchHandler from src.tools.web_search_factory import create_web_search_tool # Create search handler with tools tools = [] + # Add biomedical search tools (always available, no API keys required) + tools.append(PubMedTool()) + logger.info("PubMed tool added to search handler") + + tools.append(ClinicalTrialsTool()) + logger.info("ClinicalTrials tool added to search handler") + + tools.append(EuropePMCTool()) + logger.info("EuropePMC tool added to search handler") + # Add web search tool web_search_tool = create_web_search_tool(provider=web_search_provider or "auto") if web_search_tool: diff --git a/src/tools/fallback_web_search.py b/src/tools/fallback_web_search.py new file mode 100644 index 00000000..c91fd553 --- /dev/null +++ b/src/tools/fallback_web_search.py @@ -0,0 +1,117 @@ +"""Fallback web search tool that tries Serper first, then DuckDuckGo on errors.""" + +import structlog + +from src.tools.serper_web_search import SerperWebSearchTool +from src.tools.web_search import WebSearchTool +from src.utils.config import settings +from src.utils.exceptions import ConfigurationError, RateLimitError, SearchError +from src.utils.models import Evidence + +logger = structlog.get_logger() + + +class FallbackWebSearchTool: + """Web search tool that tries Serper first, falls back to DuckDuckGo on any error. + + This ensures search always works even if Serper fails due to: + - Credit exhaustion (403 Forbidden) + - Rate limiting (429) + - Network errors + - Invalid API key + - Any other errors + """ + + def __init__(self) -> None: + """Initialize fallback web search tool.""" + self._serper_tool: SerperWebSearchTool | None = None + self._duckduckgo_tool: WebSearchTool | None = None + self._serper_available = False + + # Try to initialize Serper if API key is available + if settings.serper_api_key: + try: + self._serper_tool = SerperWebSearchTool() + self._serper_available = True + logger.info("Serper web search initialized for fallback tool") + except Exception as e: + logger.warning( + "Failed to initialize Serper, will use DuckDuckGo only", + error=str(e), + ) + self._serper_available = False + + # DuckDuckGo is always available as fallback + self._duckduckgo_tool = WebSearchTool() + logger.info("DuckDuckGo web search initialized as fallback") + + @property + def name(self) -> str: + """Return the name of this search tool.""" + return "serper" if self._serper_available else "duckduckgo" + + async def search(self, query: str, max_results: int = 10) -> list[Evidence]: + """Execute web search with automatic fallback. + + Args: + query: The search query string + max_results: Maximum number of results to return + + Returns: + List of Evidence objects from Serper (if successful) or DuckDuckGo (if fallback) + """ + # Try Serper first if available + if self._serper_available and self._serper_tool: + try: + logger.debug("Attempting Serper search", query=query) + results = await self._serper_tool.search(query, max_results=max_results) + logger.info( + "Serper search successful", + query=query, + results_count=len(results), + ) + return results + except (ConfigurationError, RateLimitError, SearchError) as e: + # Serper failed - log and fall back to DuckDuckGo + logger.warning( + "Serper search failed, falling back to DuckDuckGo", + error=str(e), + error_type=type(e).__name__, + query=query, + ) + # Mark Serper as unavailable for future requests (optional optimization) + # self._serper_available = False + except Exception as e: + # Unexpected error from Serper - fall back + logger.error( + "Unexpected error in Serper search, falling back to DuckDuckGo", + error=str(e), + error_type=type(e).__name__, + query=query, + ) + + # Fall back to DuckDuckGo + if self._duckduckgo_tool: + try: + logger.info("Using DuckDuckGo search", query=query) + results = await self._duckduckgo_tool.search(query, max_results=max_results) + logger.info( + "DuckDuckGo search successful", + query=query, + results_count=len(results), + ) + return results + except Exception as e: + logger.error( + "DuckDuckGo search also failed", + error=str(e), + query=query, + ) + # If even DuckDuckGo fails, return empty list + return [] + + # Should never reach here, but just in case + logger.error("No web search tools available") + return [] + + diff --git a/src/tools/query_utils.py b/src/tools/query_utils.py index 3a0b9681..c52507f2 100644 --- a/src/tools/query_utils.py +++ b/src/tools/query_utils.py @@ -159,3 +159,60 @@ def preprocess_query(raw_query: str) -> str: query = expand_synonyms(query) return query.strip() + + +def preprocess_web_query(raw_query: str) -> str: + """ + Simplified preprocessing pipeline for web search engines (Serper, DuckDuckGo, etc.). + + Web search engines work better with natural language queries rather than + complex boolean syntax. This function: + 1. Strips whitespace and punctuation + 2. Removes question words (less aggressively) + 3. Removes complex boolean syntax (OR groups, parentheses) + 4. Uses primary synonym terms instead of expanding to OR groups + + Args: + raw_query: Natural language query from user + + Returns: + Simplified query optimized for web search engines + """ + if not raw_query or not raw_query.strip(): + return "" + + # Remove question marks and extra whitespace + query = raw_query.replace("?", "").strip() + query = re.sub(r"\s+", " ", query) + + # Remove complex boolean syntax that might have been added + # Remove OR groups like: ("term1" OR "term2" OR "term3") + query = re.sub(r'\([^)]*OR[^)]*\)', '', query, flags=re.IGNORECASE) + # Remove standalone OR statements + query = re.sub(r'\s+OR\s+', ' ', query, flags=re.IGNORECASE) + # Remove extra parentheses + query = re.sub(r'[()]', '', query) + # Remove extra quotes that might be left + query = re.sub(r'"([^"]*)"', r'\1', query) + # Clean up multiple spaces + query = re.sub(r'\s+', ' ', query) + + # Strip question words (less aggressively - keep important context words) + # Only remove very common question starters + minimal_question_words = {"what", "which", "how", "why", "when", "where", "who"} + words = query.split() + filtered = [w for w in words if w.lower() not in minimal_question_words] + query = " ".join(filtered) + + # Replace known medical terms with their primary/common form + # Use the first synonym (most common) instead of OR groups + query_lower = query.lower() + for term, expansions in SYNONYMS.items(): + if term in query_lower: + # Use the first expansion (usually the most common term) + primary_term = expansions[0] if expansions else term + # Replace case-insensitively, preserving original case where possible + pattern = re.compile(re.escape(term), re.IGNORECASE) + query = pattern.sub(primary_term, query) + + return query.strip() \ No newline at end of file diff --git a/src/tools/rate_limiter.py b/src/tools/rate_limiter.py index 61e0afdc..48cf5385 100644 --- a/src/tools/rate_limiter.py +++ b/src/tools/rate_limiter.py @@ -1,6 +1,7 @@ """Rate limiting utilities using the limits library.""" import asyncio +import random from typing import ClassVar from limits import RateLimitItem, parse @@ -28,7 +29,7 @@ def __init__(self, rate: str) -> None: self._rate_limit: RateLimitItem = parse(rate) self._identity = "default" # Single identity for shared limiting - async def acquire(self, wait: bool = True) -> bool: + async def acquire(self, wait: bool = True, jitter: bool = False) -> bool: """ Acquire permission to make a request. @@ -37,6 +38,7 @@ async def acquire(self, wait: bool = True) -> bool: Args: wait: If True, wait until allowed. If False, return immediately. + jitter: If True, add random jitter (0-20% of wait time) to avoid thundering herd Returns: True if allowed, False if not (only when wait=False) @@ -44,6 +46,12 @@ async def acquire(self, wait: bool = True) -> bool: while True: # Check if we can proceed (synchronous, fast - ~microseconds) if self._limiter.hit(self._rate_limit, self._identity): + # Add jitter after acquiring to spread out requests + if jitter: + # Add 0-1 second jitter to spread requests slightly + # This prevents thundering herd without long delays + jitter_seconds = random.uniform(0, 1.0) + await asyncio.sleep(jitter_seconds) return True if not wait: @@ -97,7 +105,15 @@ def get_serper_limiter(api_key: str | None = None) -> RateLimiter: """ Get the shared Serper API rate limiter. - Rate: 10 requests/second (Serper API limit) + Rate: 100 requests/second (Serper free tier limit) + + Serper free tier provides: + - 2,500 credits (one-time, expire after 6 months) + - 100 requests/second rate limit + - Credits only deduct for successful responses + + We use a slightly conservative rate (90/second) to stay safely under the limit + while allowing high throughput when needed. Args: api_key: Serper API key (optional, for consistency with other limiters) @@ -105,7 +121,8 @@ def get_serper_limiter(api_key: str | None = None) -> RateLimiter: Returns: Shared RateLimiter instance """ - return RateLimiterFactory.get("serper", "10/second") + # Use 90/second to stay safely under 100/second limit + return RateLimiterFactory.get("serper", "90/second") def get_searchxng_limiter() -> RateLimiter: diff --git a/src/tools/serper_web_search.py b/src/tools/serper_web_search.py index 77fc6eb0..d5b6bdb3 100644 --- a/src/tools/serper_web_search.py +++ b/src/tools/serper_web_search.py @@ -1,9 +1,14 @@ """Serper web search tool using Serper API for Google searches.""" import structlog -from tenacity import retry, stop_after_attempt, wait_exponential - -from src.tools.query_utils import preprocess_query +from tenacity import ( + retry, + retry_if_not_exception_type, + stop_after_attempt, + wait_random_exponential, +) + +from src.tools.query_utils import preprocess_web_query from src.tools.rate_limiter import get_serper_limiter from src.tools.vendored.serper_client import SerperClient from src.tools.vendored.web_search_core import scrape_urls @@ -34,6 +39,14 @@ def __init__(self, api_key: str | None = None) -> None: self._client = SerperClient(api_key=self.api_key) self._limiter = get_serper_limiter(self.api_key) + + # Validate API key format (basic check) + if self.api_key and len(self.api_key.strip()) < 10: + logger.warning( + "Serper API key appears to be too short", + key_length=len(self.api_key), + hint="Verify SERPER_API_KEY is correct", + ) @property def name(self) -> str: @@ -41,13 +54,20 @@ def name(self) -> str: return "serper" async def _rate_limit(self) -> None: - """Enforce Serper API rate limiting.""" - await self._limiter.acquire() + """Enforce Serper API rate limiting with jitter. + + Uses jitter to spread out requests and avoid thundering herd problems. + Rate limit is 100 requests/second for free tier, we use 90/second to stay safe. + """ + await self._limiter.acquire(jitter=True) @retry( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=1, max=10), + stop=stop_after_attempt(3), # Reduced retries for faster fallback + wait=wait_random_exponential( + multiplier=1, min=2, max=10, exp_base=2 + ), # 2s to 10s backoff with jitter (faster for fallback) reraise=True, + retry=retry_if_not_exception_type(ConfigurationError), # Don't retry on config errors ) async def search(self, query: str, max_results: int = 10) -> list[Evidence]: """Execute a web search using Serper API. @@ -62,11 +82,12 @@ async def search(self, query: str, max_results: int = 10) -> list[Evidence]: Raises: SearchError: If the search fails RateLimitError: If rate limit is exceeded + ConfigurationError: If API key is invalid (403 Forbidden) """ await self._rate_limit() - # Preprocess query to remove noise - clean_query = preprocess_query(query) + # Preprocess query for web search (simplified, no boolean syntax) + clean_query = preprocess_web_query(query) final_query = clean_query if clean_query else query try: @@ -111,6 +132,9 @@ async def search(self, query: str, max_results: int = 10) -> list[Evidence]: return evidence + except ConfigurationError: + # Don't retry configuration errors (e.g., 403 Forbidden = invalid API key) + raise except RateLimitError: raise except SearchError: diff --git a/src/tools/vendored/serper_client.py b/src/tools/vendored/serper_client.py index 6b54c10b..17d30b6a 100644 --- a/src/tools/vendored/serper_client.py +++ b/src/tools/vendored/serper_client.py @@ -9,7 +9,7 @@ import structlog from src.tools.vendored.web_search_core import WebpageSnippet, ssl_context -from src.utils.exceptions import RateLimitError, SearchError +from src.utils.exceptions import ConfigurationError, RateLimitError, SearchError logger = structlog.get_logger() @@ -34,6 +34,11 @@ def __init__(self, api_key: str | None = None) -> None: "No API key provided. Set SERPER_API_KEY environment variable." ) + # Serper API endpoint and headers + # Documentation: https://serper.dev/api + # Format: POST https://google.serper.dev/search + # Headers: X-API-KEY (required), Content-Type: application/json + # Body: {"q": "search query", "autocorrect": false} self.url = "https://google.serper.dev/search" self.headers = {"X-API-KEY": self.api_key, "Content-Type": "application/json"} @@ -57,11 +62,43 @@ async def search( connector = aiohttp.TCPConnector(ssl=ssl_context) try: async with aiohttp.ClientSession(connector=connector) as session: + # Verify API call format matches Serper API documentation: + # POST https://google.serper.dev/search + # Headers: X-API-KEY, Content-Type: application/json + # Body: {"q": query, "autocorrect": false} async with session.post( - self.url, headers=self.headers, json={"q": query, "autocorrect": False} + self.url, + headers=self.headers, + json={"q": query, "autocorrect": False}, + timeout=aiohttp.ClientTimeout(total=30), # 30 second timeout ) as response: if response.status == 429: raise RateLimitError("Serper API rate limit exceeded") + + if response.status == 403: + # 403 can mean either invalid key OR credits exhausted + # For free tier (2,500 credits), it's often credit exhaustion + # Read response body to get more details + try: + error_body = await response.text() + logger.warning( + "Serper API returned 403 Forbidden", + status=403, + body=error_body[:200], # Truncate for logging + hint="May be credit exhaustion (free tier: 2,500 credits) or invalid key", + ) + except Exception: + pass + + # Raise RateLimitError instead of ConfigurationError + # This allows retry logic to handle credit exhaustion + # The retry decorator will use exponential backoff with jitter + raise RateLimitError( + "Serper API credits may be exhausted (403 Forbidden). " + "Free tier provides 2,500 credits (one-time, expire after 6 months). " + "Check your dashboard at https://serper.dev/dashboard. " + "Retrying with backoff..." + ) response.raise_for_status() results = await response.json() diff --git a/src/tools/web_search_factory.py b/src/tools/web_search_factory.py index 213de5de..ae798116 100644 --- a/src/tools/web_search_factory.py +++ b/src/tools/web_search_factory.py @@ -3,6 +3,7 @@ import structlog from src.tools.base import SearchTool +from src.tools.fallback_web_search import FallbackWebSearchTool from src.tools.searchxng_web_search import SearchXNGWebSearchTool from src.tools.serper_web_search import SerperWebSearchTool from src.tools.web_search import WebSearchTool @@ -37,17 +38,18 @@ def create_web_search_tool(provider: str | None = None) -> SearchTool | None: # Auto-detect best available provider if "auto" or if provider is duckduckgo but better options exist if provider == "auto" or (provider == "duckduckgo" and settings.serper_api_key): - # Prefer Serper if API key is available (better quality) + # Use fallback tool if Serper API key is available + # This automatically falls back to DuckDuckGo on any Serper error if settings.serper_api_key: try: logger.info( - "Auto-detected Serper web search (SERPER_API_KEY found)", - provider="serper", + "Auto-detected Serper with DuckDuckGo fallback (SERPER_API_KEY found)", + provider="serper+duckduckgo", ) - return SerperWebSearchTool() + return FallbackWebSearchTool() except Exception as e: logger.warning( - "Failed to initialize Serper, falling back", + "Failed to initialize fallback web search, trying alternatives", error=str(e), ) @@ -65,7 +67,7 @@ def create_web_search_tool(provider: str | None = None) -> SearchTool | None: error=str(e), ) - # Fall back to DuckDuckGo + # Fall back to DuckDuckGo only if provider == "auto": logger.info( "Auto-detected DuckDuckGo web search (no API keys found)", From bbab90cbcf51b9e0900ed0fa5fc79b95b71dbe75 Mon Sep 17 00:00:00 2001 From: Joseph Pollack Date: Mon, 8 Dec 2025 14:14:22 +0100 Subject: [PATCH 2/2] fix typos and errors --- docs/troubleshooting/serper-403-error.md | 1 + src/app.py | 6 ++++++ src/orchestrator/research_flow.py | 2 +- src/tools/fallback_web_search.py | 1 + src/tools/search_handler.py | 2 ++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/troubleshooting/serper-403-error.md b/docs/troubleshooting/serper-403-error.md index 005080b9..01d76610 100644 --- a/docs/troubleshooting/serper-403-error.md +++ b/docs/troubleshooting/serper-403-error.md @@ -202,3 +202,4 @@ This will use DuckDuckGo instead of Serper, allowing your research to continue ( + diff --git a/src/app.py b/src/app.py index 68007d89..ba6795ef 100644 --- a/src/app.py +++ b/src/app.py @@ -83,6 +83,7 @@ def configure_orchestrator( """ from src.tools.clinicaltrials import ClinicalTrialsTool from src.tools.europepmc import EuropePMCTool + from src.tools.neo4j_search import Neo4jSearchTool from src.tools.pubmed import PubMedTool from src.tools.search_handler import SearchHandler from src.tools.web_search_factory import create_web_search_tool @@ -99,6 +100,11 @@ def configure_orchestrator( tools.append(EuropePMCTool()) logger.info("EuropePMC tool added to search handler") + + # Add Neo4j knowledge graph search tool (if Neo4j is configured) + neo4j_tool = Neo4jSearchTool() + tools.append(neo4j_tool) + logger.info("Neo4j search tool added to search handler") # Add web search tool web_search_tool = create_web_search_tool(provider=web_search_provider or "auto") diff --git a/src/orchestrator/research_flow.py b/src/orchestrator/research_flow.py index 3ce777bd..3ae58be1 100644 --- a/src/orchestrator/research_flow.py +++ b/src/orchestrator/research_flow.py @@ -522,7 +522,7 @@ def _get_rag_service(self) -> LlamaIndexRAGService | None: """ if self._rag_service is None: try: - self._rag_service = get_rag_service() + self._rag_service = get_rag_service(oauth_token=self.oauth_token) self.logger.info("RAG service initialized for research flow") except (ConfigurationError, ImportError) as e: self.logger.warning( diff --git a/src/tools/fallback_web_search.py b/src/tools/fallback_web_search.py index c91fd553..62d2dfd0 100644 --- a/src/tools/fallback_web_search.py +++ b/src/tools/fallback_web_search.py @@ -115,3 +115,4 @@ async def search(self, query: str, max_results: int = 10) -> list[Evidence]: return [] + diff --git a/src/tools/search_handler.py b/src/tools/search_handler.py index 8814a8b7..9d2b4adf 100644 --- a/src/tools/search_handler.py +++ b/src/tools/search_handler.py @@ -118,6 +118,7 @@ async def execute(self, query: str, max_results_per_tool: int = 10) -> SearchRes "pubmed": "pubmed", "clinicaltrials": "clinicaltrials", "europepmc": "europepmc", + "neo4j": "neo4j", "rag": "rag", "web": "web", # In case tool already uses "web" } @@ -141,6 +142,7 @@ async def execute(self, query: str, max_results_per_tool: int = 10) -> SearchRes "preprint", "rag", "web", + "neo4j", ]: logger.warning( "Tool name not in SourceName literals, defaulting to 'web'",