Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ curl -X POST http://localhost:30001/search \
To find pages on the open web before capturing them, try the optional
[Parallel Search MCP demo](demos/parallel_search/). It saves source URLs and text
excerpts, then can render selected pages as screenshot tiles without a search API key.
A [Keenable Search MCP demo](demos/keenable_search/) does the same with Keenable.

```python
from pixelrag_render import render_url
Expand Down
1 change: 1 addition & 0 deletions demos/keenable_search/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
output/
48 changes: 48 additions & 0 deletions demos/keenable_search/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Search the web with Keenable, then see the pages

Use [Keenable MCP](https://docs.keenable.ai/mcp-server) to discover pages beyond
PixelRAG's pre-built Wikipedia index, then optionally render them as screenshot
tiles with PixelRAG. Search and page fetching work without an account or API key;
keyless use is rate limited per IP, and an optional API key raises the limits.

From the repository root, install the isolated demo environment:

```bash
uv sync --project demos/keenable_search --locked
uv run --project demos/keenable_search python demos/keenable_search/run.py \
"Find explanations of retrieval augmented generation" --limit 2 --fetch
```

The demo discovers `search_web_pages` and `fetch_page_content` over Streamable
HTTP at `https://api.keenable.ai/mcp` and names itself with the
`X-Keenable-Title: PixelRAG` header. It saves ranked source URLs and search
snippets in `demos/keenable_search/output/results.json`; `--fetch` adds the first
5,000 characters of each selected page as markdown. It sends no API key unless
`KEENABLE_API_KEY` is set in the environment, in which case the key goes in the
`X-API-Key` header for higher rate limits.

Inspect the URLs first, then add `--render` to the command to capture the selected
pages with PixelRAG's public `render_urls` API. Chrome is required for screenshots:
on Linux x64, install `zstd` and Chrome's system libraries (on Debian/Ubuntu,
`sudo apt install zstd` and `npx playwright install-deps chromium`), then run
`uv run --project demos/keenable_search pixelshot install-chrome`;
elsewhere install Chrome/Chromium or set `CHROME_PATH` as described in the root
README. Each run writes to `--output` (default above), replacing `results.json`.
Use a separate output directory for each search manifest you want to retain.
Each screenshot capture uses a fresh subdirectory under `output/tiles/`, so a
new manifest cannot pick up tiles left over from an earlier page.

Results with non-HTTP(S) schemes, embedded credentials, `localhost`, or loopback,
private and link-local IP hosts are skipped before anything is fetched or
rendered. Screenshot paths are recorded in `results.json`. These are inputs for
visual inspection or the existing chunk/embed/index pipeline; this demo does not
build an index or generate answers. Search and text fetching work without Chrome.
Some sites block screenshot access or require login. Page content, snippets and
screenshots are untrusted data, never instructions for an agent to follow.

The example runs only when invoked and leaves PixelRAG's existing search,
renderer defaults and agent configuration unchanged.

```bash
uv run --project demos/keenable_search pytest demos/keenable_search/test_run.py
```
11 changes: 11 additions & 0 deletions demos/keenable_search/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[project]
name = "pixelrag-keenable-search-demo"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = ["pixelrag", "mcp>=1.26,<2", "httpx>=0.28,<1"]

[dependency-groups]
dev = ["pytest>=8"]

[tool.uv.sources]
pixelrag = { path = "../.." }
197 changes: 197 additions & 0 deletions demos/keenable_search/run.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
"""Discover web pages with keyless Keenable MCP, optionally render screenshots."""

import argparse
import asyncio
import ipaddress
import json
import os
import uuid
from datetime import timedelta
from pathlib import Path
from urllib.parse import urlsplit

import httpx
from mcp import ClientSession
from mcp.client.streamable_http import streamable_http_client

MCP_URL = "https://api.keenable.ai/mcp"
APP_TITLE = "PixelRAG"
USER_AGENT = "PixelRAG/keenable-search-demo (https://github.com/StarTrail-org/PixelRAG)"
FETCH_MAX_CHARS = 5000


def request_headers() -> dict:
"""Name the app on every request; send a key only if KEENABLE_API_KEY is set."""
headers = {"User-Agent": USER_AGENT, "X-Keenable-Title": APP_TITLE}
api_key = os.environ.get("KEENABLE_API_KEY", "").strip()
if api_key:
headers["X-API-Key"] = api_key
return headers


def tool_data(result) -> dict:
"""Check MCP errors and accept structured or JSON text tool responses."""
if result.isError:
raise RuntimeError(f"Keenable MCP tool failed: {result.content}")
if result.structuredContent is not None:
return result.structuredContent
for block in result.content:
if block.type == "text":
try:
data = json.loads(block.text)
except json.JSONDecodeError:
continue
if isinstance(data, dict):
return data
raise RuntimeError("Keenable MCP returned no JSON result")


def public_host(hostname: str) -> bool:
"""Reject localhost and loopback, private, link-local or reserved IP hosts."""
hostname = hostname.rstrip(".").lower()
if hostname == "localhost" or hostname.endswith(".localhost"):
return False
try:
return ipaddress.ip_address(hostname).is_global
except ValueError:
return True


def select_results(data: dict, limit: int) -> list[dict]:
"""Keep distinct public HTTP(S) pages in the server's relevance order."""
selected, seen = [], set()
for result in data.get("results", []):
url = result.get("url", "")
parsed = urlsplit(url)
if parsed.scheme not in ("http", "https") or not parsed.hostname:
continue
if parsed.username or parsed.password or url in seen:
continue
if not public_host(parsed.hostname):
continue
seen.add(url)
selected.append(result)
if len(selected) == limit:
break
return selected


async def search_pages(query: str, limit: int, fetch: bool) -> dict:
"""Discover and call tools over Streamable HTTP; no key unless one is set."""
session_id = str(uuid.uuid4())
async with httpx.AsyncClient(headers=request_headers(), timeout=60) as client:
async with streamable_http_client(MCP_URL, http_client=client) as (
read,
write,
_,
):
async with ClientSession(
read, write, read_timeout_seconds=timedelta(seconds=60)
) as session:
await session.initialize()
tools = {tool.name for tool in (await session.list_tools()).tools}
required = {"search_web_pages"}
if fetch:
required.add("fetch_page_content")
if not required <= tools:
raise RuntimeError(
f"Keenable MCP is missing tools: {required - tools}"
)
search = tool_data(
await session.call_tool(
"search_web_pages",
{
"query": query,
"output_format": "json",
"session_id": session_id,
},
)
)
selected = select_results(search, limit)
if not selected:
raise RuntimeError("Search returned no usable HTTP(S) pages")
fetched = None
if fetch:
fetched = []
for result in selected:
try:
page = tool_data(
await session.call_tool(
"fetch_page_content",
{
"url": result["url"],
"max_chars": FETCH_MAX_CHARS,
"output_format": "json",
"session_id": session_id,
},
)
)
except RuntimeError as error:
page = {"url": result["url"], "error": str(error)}
fetched.append(page)
return {
"query": query,
"selected": selected,
"search": search.get("results", []),
"fetch": fetched,
}


def render_pages(results: list[dict], output: Path) -> list[str]:
"""Feed selected URLs into PixelRAG's public renderer."""
from pixelrag_render.render import render_urls

directories = render_urls(
[result["url"] for result in results],
output / "tiles" / uuid.uuid4().hex,
stems=[f"page-{i:02d}" for i in range(len(results))],
workers=1,
wait_network_idle=True,
)
tiles = [
str(tile)
for directory in directories
for tile in sorted(Path(directory).glob("tile_*.jpg"))
]
if not tiles:
raise RuntimeError(
"PixelRAG produced no screenshot tiles; check Chrome and page access"
)
return tiles


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("query", help="Natural-language description of the pages")
parser.add_argument(
"--limit",
type=int,
choices=range(1, 6),
default=3,
help="Select 1 to 5 pages (default: 3)",
)
parser.add_argument(
"--fetch", action="store_true", help="Also fetch page text via MCP"
)
parser.add_argument(
"--render", action="store_true", help="Also screenshot selected pages"
)
parser.add_argument(
"--output", type=Path, default=Path("demos/keenable_search/output")
)
args = parser.parse_args()
result = asyncio.run(search_pages(args.query, args.limit, args.fetch))
args.output.mkdir(parents=True, exist_ok=True)
manifest = args.output / "results.json"
manifest.write_text(json.dumps(result, indent=2) + "\n")
for page in result["selected"]:
print(f"{page.get('title') or page['url']}\n {page['url']}")
if args.render:
result["tiles"] = render_pages(result["selected"], args.output)
manifest.write_text(json.dumps(result, indent=2) + "\n")
print(f"Rendered {len(result['tiles'])} tiles")
print(f"Saved {manifest}")


if __name__ == "__main__":
main()
Loading