diff --git a/assets/api-banner.png b/assets/api-banner.png new file mode 100644 index 00000000..06de517d Binary files /dev/null and b/assets/api-banner.png differ diff --git a/docs/img/apikey_1.png b/assets/apikey_1.png similarity index 100% rename from docs/img/apikey_1.png rename to assets/apikey_1.png diff --git a/docs/img/apikey_2.png b/assets/apikey_2.png similarity index 100% rename from docs/img/apikey_2.png rename to assets/apikey_2.png diff --git a/docs/img/apikey_3.png b/assets/apikey_3.png similarity index 100% rename from docs/img/apikey_3.png rename to assets/apikey_3.png diff --git a/docs/img/apikey_4.png b/assets/apikey_4.png similarity index 100% rename from docs/img/apikey_4.png rename to assets/apikey_4.png diff --git a/assets/browserbase_logo.png b/assets/browserbase_logo.png new file mode 100644 index 00000000..bd16f2e1 Binary files /dev/null and b/assets/browserbase_logo.png differ diff --git a/assets/browserbase_logo.svg b/assets/browserbase_logo.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/browserbase_logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/codespaces-badge.png b/assets/codespaces-badge.png new file mode 100644 index 00000000..2743b211 Binary files /dev/null and b/assets/codespaces-badge.png differ diff --git a/assets/logo_authors.png b/assets/logo_authors.png new file mode 100644 index 00000000..116cba46 Binary files /dev/null and b/assets/logo_authors.png differ diff --git a/assets/omniscrapergraph.png b/assets/omniscrapergraph.png new file mode 100644 index 00000000..e1426039 Binary files /dev/null and b/assets/omniscrapergraph.png differ diff --git a/assets/omniscrapergraph.svg b/assets/omniscrapergraph.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/omniscrapergraph.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/omnisearchgraph.png b/assets/omnisearchgraph.png new file mode 100644 index 00000000..f2ab22d6 Binary files /dev/null and b/assets/omnisearchgraph.png differ diff --git a/assets/project_overview_diagram.fig b/assets/project_overview_diagram.fig new file mode 100644 index 00000000..c5fa03d4 Binary files /dev/null and b/assets/project_overview_diagram.fig differ diff --git a/assets/project_overview_diagram.png b/assets/project_overview_diagram.png new file mode 100644 index 00000000..cf4d2a18 Binary files /dev/null and b/assets/project_overview_diagram.png differ diff --git a/assets/project_overview_diagram.svg b/assets/project_overview_diagram.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/project_overview_diagram.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/scrapedo.png b/assets/scrapedo.png new file mode 100644 index 00000000..d15ea04e Binary files /dev/null and b/assets/scrapedo.png differ diff --git a/assets/scrapedo.svg b/assets/scrapedo.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/scrapedo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/scrapegraphai_logo.png b/assets/scrapegraphai_logo.png new file mode 100644 index 00000000..ca24928b Binary files /dev/null and b/assets/scrapegraphai_logo.png differ diff --git a/assets/scrapegraphai_logo.svg b/assets/scrapegraphai_logo.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/scrapegraphai_logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/scriptcreatorgraph.png b/assets/scriptcreatorgraph.png new file mode 100644 index 00000000..e70197b9 Binary files /dev/null and b/assets/scriptcreatorgraph.png differ diff --git a/assets/searchgraph.png b/assets/searchgraph.png new file mode 100644 index 00000000..ab841b1d Binary files /dev/null and b/assets/searchgraph.png differ diff --git a/assets/serp_api_logo.png b/assets/serp_api_logo.png new file mode 100644 index 00000000..ff2f1b01 Binary files /dev/null and b/assets/serp_api_logo.png differ diff --git a/assets/serp_api_logo.svg b/assets/serp_api_logo.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/serp_api_logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/sgai-hero.png b/assets/sgai-hero.png new file mode 100644 index 00000000..e432d2da Binary files /dev/null and b/assets/sgai-hero.png differ diff --git a/assets/smartscrapergraph.png b/assets/smartscrapergraph.png new file mode 100644 index 00000000..54707f8e Binary files /dev/null and b/assets/smartscrapergraph.png differ diff --git a/assets/speechgraph.png b/assets/speechgraph.png new file mode 100644 index 00000000..e61c0346 Binary files /dev/null and b/assets/speechgraph.png differ diff --git a/assets/transparent_stat.png b/assets/transparent_stat.png new file mode 100644 index 00000000..d4f3c5fc Binary files /dev/null and b/assets/transparent_stat.png differ diff --git a/assets/transparent_stat.svg b/assets/transparent_stat.svg new file mode 100644 index 00000000..0519ecba --- /dev/null +++ b/assets/transparent_stat.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/Builders/GraphBuilder.md b/docs/Builders/GraphBuilder.md deleted file mode 100644 index 584ae4d1..00000000 --- a/docs/Builders/GraphBuilder.md +++ /dev/null @@ -1,86 +0,0 @@ -# ๐Ÿ’ซ Graph Builder -# GraphBuilder Module - -The `GraphBuilder` module constructs web scraping graphs based on user prompts. It uses a natural language understanding model to interpret user prompts and generate a graph configuration for scraping web content. - -## Classes - -### `GraphBuilder` - -`GraphBuilder` is a dynamic tool for constructing web scraping graphs based on user prompts. - -#### Attributes - -- **prompt (str)**: The user's natural language prompt for the scraping task. -- **llm (ChatOpenAI)**: An instance of the `ChatOpenAI` class configured with the specified `llm_config`. -- **nodes_description (str)**: A string description of all available nodes and their arguments. -- **chain (LLMChain)**: The extraction chain responsible for processing the prompt and creating the graph. - -#### Methods - -- **`__init__(self, user_prompt: str, config: dict)`** - - Initializes the `GraphBuilder` with a user prompt and language model configuration. - -- **`_create_llm(self, llm_config: dict)`** - - Creates an instance of the `OpenAI` class with the provided language model configuration. - - **Returns**: `OpenAI` - An instance of the `OpenAI` class. - - **Raises**: `ValueError` - If 'api_key' is not provided in `llm_config`. - -- **`_generate_nodes_description(self)`** - - Generates a string description of all available nodes and their arguments. - - **Returns**: `str` - A string description of all available nodes and their arguments. - -- **`_create_extraction_chain(self)`** - - Creates an extraction chain for processing the user prompt and generating the graph configuration. - - **Returns**: `LLMChain` - An instance of the `LLMChain` class. - -- **`build_graph(self)`** - - Executes the graph creation process based on the user prompt and returns the graph configuration. - - **Returns**: `dict` - A JSON representation of the graph configuration. - -- **`convert_json_to_graphviz(json_data, format: str = 'pdf')`** - - Converts a JSON graph configuration to a Graphviz object for visualization. - - **Args**: - - `json_data (dict)`: A JSON representation of the graph configuration. - - `format (str)`: The output format of the Graphviz object. Defaults to 'pdf'. - - **Returns**: `graphviz.Digraph` - A Graphviz object representing the graph configuration. - -#### Args - -- **prompt (str)**: The user's natural language prompt describing the desired scraping operation. -- **url (str)**: The target URL from which data is to be scraped. -- **llm_config (dict)**: Configuration parameters for the language model, where 'api_key' is mandatory, and 'model_name', 'temperature', and 'streaming' can be optionally included. - -#### Raises - -- **ValueError**: If 'api_key' is not included in `llm_config`. - -## Example Usage - -Here is an example of how to use the `GraphBuilder` class: - -```python -from scrapegraphai.builders.graph_builder import GraphBuilder - -# Define user prompt and configuration -user_prompt = "Scrape all article titles and publish dates from the homepage" -config = { - "llm": { - "api_key": "your_api_key_here", - "model_name": "gpt-3.5-turbo", - "temperature": 0.7, - "streaming": False - } -} - -# Initialize GraphBuilder -graph_builder = GraphBuilder(user_prompt, config) - -# Build the graph -graph_config = graph_builder.build_graph() - -# Convert graph config to Graphviz for visualization -graphviz_graph = GraphBuilder.convert_json_to_graphviz(graph_config) - -# Render the graph to a file -graphviz_graph.render(filename='graph', format='pdf') diff --git a/docs/Builders/_category_.json b/docs/Builders/_category_.json deleted file mode 100644 index 34ce9025..00000000 --- a/docs/Builders/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Builders", - "position": 4, - "link": { - "type": "generated-index", - "description": "Introduction graph builder" - } -} \ No newline at end of file diff --git a/docs/Contributing/contributing_guidelines.md b/docs/Contributing/contributing_guidelines.md index da42f379..4932857a 100644 --- a/docs/Contributing/contributing_guidelines.md +++ b/docs/Contributing/contributing_guidelines.md @@ -1,10 +1,27 @@ -# ๐Ÿค Contributing -Scrapegraph-ai is [MIT LICENSED](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/LICENSE). +# ๐Ÿš€ Contributing guidelines -Contributions are welcome! Please check out the todos below, and feel free to open a pull request. +Hey there! Thanks for checking out **ScrapeGraphAI**! We're excited to have you here! ๐ŸŽ‰ -For more information, please see the [contributing guidelines](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/CONTRIBUTING.md). +## Quick Start Guide ๐Ÿƒโ€โ™‚๏ธ -Join our Discord server to discuss with us improvements and give us suggestions! +1. Fork the repository from the **pre/beta branch** ๐Ÿด +2. Clone your fork locally ๐Ÿ’ป +3. Install uv (if you haven't): + ```bash + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` +4. Run `uv sync` (creates virtual env & installs dependencies) โšก +5. Run `uv run pre-commit install` ๐Ÿ”ง +6. Make your awesome changes โœจ +7. Test thoroughly ๐Ÿงช +8. Push & open a PR to the pre/beta branch ๐ŸŽฏ -[![Join Discord Server](https://img.shields.io/badge/Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.com/invite/gkxQDAjfeX) \ No newline at end of file +## Need Help? ๐Ÿค” + +Found a bug or have a cool idea? Open an issue and let's chat! ๐Ÿ’ฌ + +## License ๐Ÿ“œ + +MIT Licensed. See [LICENSE](LICENSE) file for details. + +Let's build something amazing together! ๐ŸŒŸ \ No newline at end of file diff --git a/docs/Contributing/sponsors.md b/docs/Contributing/sponsors.md index 3bc3cdb9..3725fb87 100644 --- a/docs/Contributing/sponsors.md +++ b/docs/Contributing/sponsors.md @@ -1,23 +1,16 @@ -
+# ๐Ÿ† Sponsors + +
- Browserbase + Browserbase - SerpAPI + SerpAPI - Stats + Stats -
+ + Stats + +
\ No newline at end of file diff --git a/docs/Docloaders/Docloader.md b/docs/Docloaders/Docloader.md deleted file mode 100644 index f9ea385a..00000000 --- a/docs/Docloaders/Docloader.md +++ /dev/null @@ -1,69 +0,0 @@ -# ChromiumLoader Module - -The `ChromiumLoader` module scrapes HTML pages from URLs using a (headless) instance of the Chromium web driver with proxy protection. - -## Classes - -### `ChromiumLoader` - -`ChromiumLoader` is designed to scrape HTML pages from a list of URLs using a specified web driver backend, such as Playwright. - -#### Attributes - -- **backend (str)**: The web driver backend library; defaults to 'playwright'. -- **browser_config (dict)**: A dictionary containing additional browser kwargs. -- **headless (bool)**: Whether to run the browser in headless mode. -- **proxy (Optional[Proxy])**: A dictionary containing proxy settings; None disables protection. -- **urls (List[str])**: A list of URLs to scrape content from. - -#### Methods - -- **`__init__(self, urls: List[str], *, backend: str = "playwright", headless: bool = True, proxy: Optional[Proxy] = None, **kwargs: Any)`** - - Initializes the loader with a list of URL paths. - - **Args**: - - `backend (str)`: The web driver backend library; defaults to 'playwright'. - - `headless (bool)`: Whether to run the browser in headless mode. - - `proxy (Optional[Proxy])`: A dictionary containing proxy information; None disables protection. - - `urls (List[str])`: A list of URLs to scrape content from. - - `kwargs (Any)`: A dictionary containing additional browser kwargs. - - **Raises**: `ImportError` - If the required backend package is not installed. - -- **`ascrape_playwright(self, url: str) -> str`** - - Asynchronously scrape the content of a given URL using Playwright's async API. - - **Args**: - - `url (str)`: The URL to scrape. - - **Returns**: `str` - The scraped HTML content or an error message if an exception occurs. - -- **`lazy_load(self) -> Iterator[Document]`** - - Lazily load text content from the provided URLs. - - **Yields**: `Document` - The scraped content encapsulated within a `Document` object. - -- **`alazy_load(self) -> AsyncIterator[Document]`** - - Asynchronously load text content from the provided URLs. - - **Yields**: `Document` - A `Document` object containing the scraped content, along with its source URL as metadata. - -## Example Usage - -Here is an example of how to use the `ChromiumLoader` class: - -```python -import asyncio -from chromium_loader import ChromiumLoader - -# Define the list of URLs to scrape -urls = ["https://example.com", "https://example.org"] - -# Initialize ChromiumLoader -loader = ChromiumLoader(urls, backend="playwright", headless=True, proxy=None) - -# Lazy load documents (synchronous) -for document in loader.lazy_load(): - print(document.page_content) - -# Asynchronously load documents -async def load_documents(): - async for document in loader.alazy_load(): - print(document.page_content) - -# Run the asynchronous loader -asyncio.run(load_documents()) diff --git a/docs/Examples/Anthropic/code_generator_graph_anthropic.md b/docs/Examples/Anthropic/code_generator_graph_anthropic.md deleted file mode 100644 index cc164e8e..00000000 --- a/docs/Examples/Anthropic/code_generator_graph_anthropic.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -anthropic_key = os.getenv("ANTHROPIC_API_KEY") - -graph_config = { - "llm": { - "api_key":anthropic_key, - "model": "anthropic/claude-3-haiku-20240307", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Anthropic/csv_scraper_anthropic.md b/docs/Examples/Anthropic/csv_scraper_anthropic.md deleted file mode 100644 index 138cfe41..00000000 --- a/docs/Examples/Anthropic/csv_scraper_anthropic.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -# required environment variables in .env -# HUGGINGFACEHUB_API_TOKEN -# ANTHROPIC_API_KEY -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Anthropic/csv_scraper_graph_multi_anthropic.md b/docs/Examples/Anthropic/csv_scraper_graph_multi_anthropic.md deleted file mode 100644 index f35e4182..00000000 --- a/docs/Examples/Anthropic/csv_scraper_graph_multi_anthropic.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Anthropic/custom_graph_anthropic.md b/docs/Examples/Anthropic/custom_graph_anthropic.md deleted file mode 100644 index f6d60c3e..00000000 --- a/docs/Examples/Anthropic/custom_graph_anthropic.md +++ /dev/null @@ -1,95 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -import os -from dotenv import load_dotenv -from langchain_anthropic import ChatAnthropic -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatAnthropic(graph_config["llm"]) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/Anthropic/depth_search_graph_anthropic.md b/docs/Examples/Anthropic/depth_search_graph_anthropic.md deleted file mode 100644 index 6d6d47b0..00000000 --- a/docs/Examples/Anthropic/depth_search_graph_anthropic.md +++ /dev/null @@ -1,29 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Anthropic/json_scraper_anthropic.md b/docs/Examples/Anthropic/json_scraper_anthropic.md deleted file mode 100644 index faed69c9..00000000 --- a/docs/Examples/Anthropic/json_scraper_anthropic.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - } -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/Anthropic/json_scraper_multi_anthropic.md b/docs/Examples/Anthropic/json_scraper_multi_anthropic.md deleted file mode 100644 index 20e411f8..00000000 --- a/docs/Examples/Anthropic/json_scraper_multi_anthropic.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Anthropic/rate_limit_anthropic.md b/docs/Examples/Anthropic/rate_limit_anthropic.md deleted file mode 100644 index 455d209f..00000000 --- a/docs/Examples/Anthropic/rate_limit_anthropic.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper while setting an API rate limit. -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - - -# required environment variables in .env -# ANTHROPIC_API_KEY -load_dotenv() - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - "rate_limit": { - "requests_per_second": 1 - } - }, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="""Don't say anything else. Output JSON only. List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, - event_end_date, event_end_time, location, event_mode, event_category, - third_party_redirect, no_of_days, - time_in_hours, hosted_or_attending, refreshments_type, - registration_available, registration_link""", - # also accepts a string with the already downloaded HTML code - source="https://www.hmhco.com/event", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Anthropic/scrape_plain_text_anthropic.md b/docs/Examples/Anthropic/scrape_plain_text_anthropic.md deleted file mode 100644 index 911a2787..00000000 --- a/docs/Examples/Anthropic/scrape_plain_text_anthropic.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Anthropic/script_generator_anthropic.md b/docs/Examples/Anthropic/script_generator_anthropic.md deleted file mode 100644 index 9006f2f9..00000000 --- a/docs/Examples/Anthropic/script_generator_anthropic.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Anthropic/script_multi_generator_anthropic.md b/docs/Examples/Anthropic/script_multi_generator_anthropic.md deleted file mode 100644 index 6c9702d4..00000000 --- a/docs/Examples/Anthropic/script_multi_generator_anthropic.md +++ /dev/null @@ -1,52 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Anthropic/search_graph_anthropic.md b/docs/Examples/Anthropic/search_graph_anthropic.md deleted file mode 100644 index e0e89407..00000000 --- a/docs/Examples/Anthropic/search_graph_anthropic.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Anthropic/search_graph_schema_anthropic.md b/docs/Examples/Anthropic/search_graph_schema_anthropic.md deleted file mode 100644 index 92630188..00000000 --- a/docs/Examples/Anthropic/search_graph_schema_anthropic.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Anthropic/search_link_graph_anthropic.md b/docs/Examples/Anthropic/search_link_graph_anthropic.md deleted file mode 100644 index 5bcd01b1..00000000 --- a/docs/Examples/Anthropic/search_link_graph_anthropic.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_openai import AzureChatOpenAI -from langchain_openai import AzureOpenAIEmbeddings - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Anthropic/smart_scraper_anthropic.md b/docs/Examples/Anthropic/smart_scraper_anthropic.md deleted file mode 100644 index 9973c82d..00000000 --- a/docs/Examples/Anthropic/smart_scraper_anthropic.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="""Don't say anything else. Output JSON only. List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, - event_end_date, event_end_time, location, event_mode, event_category, - third_party_redirect, no_of_days, - time_in_hours, hosted_or_attending, refreshments_type, - registration_available, registration_link""", - # also accepts a string with the already downloaded HTML code - source="https://www.hmhco.com/event", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Anthropic/smart_scraper_multi_anthropic.md b/docs/Examples/Anthropic/smart_scraper_multi_anthropic.md deleted file mode 100644 index d13f4f6c..00000000 --- a/docs/Examples/Anthropic/smart_scraper_multi_anthropic.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Anthropic/smart_scraper_multi_concat_anthropic.md b/docs/Examples/Anthropic/smart_scraper_multi_concat_anthropic.md deleted file mode 100644 index 426a36bb..00000000 --- a/docs/Examples/Anthropic/smart_scraper_multi_concat_anthropic.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Anthropic/smart_scraper_schema_anthropic.md b/docs/Examples/Anthropic/smart_scraper_schema_anthropic.md deleted file mode 100644 index 616d4f8b..00000000 --- a/docs/Examples/Anthropic/smart_scraper_schema_anthropic.md +++ /dev/null @@ -1,52 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - schema=Projects, - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Anthropic/xml_scraper_anthropic.md b/docs/Examples/Anthropic/xml_scraper_anthropic.md deleted file mode 100644 index 314f46d6..00000000 --- a/docs/Examples/Anthropic/xml_scraper_anthropic.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Anthropic/xml_scraper_graph_multi_anthropic.md b/docs/Examples/Anthropic/xml_scraper_graph_multi_anthropic.md deleted file mode 100644 index 0f2d9c0e..00000000 --- a/docs/Examples/Anthropic/xml_scraper_graph_multi_anthropic.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Azure/code_generator_graph_azure.md b/docs/Examples/Azure/code_generator_graph_azure.md deleted file mode 100644 index 9507b4c6..00000000 --- a/docs/Examples/Azure/code_generator_graph_azure.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Azure/csv_scraper_azure.md b/docs/Examples/Azure/csv_scraper_azure.md deleted file mode 100644 index 4108b023..00000000 --- a/docs/Examples/Azure/csv_scraper_azure.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Azure/csv_scraper_graph_multi_azure.md b/docs/Examples/Azure/csv_scraper_graph_multi_azure.md deleted file mode 100644 index 6878e389..00000000 --- a/docs/Examples/Azure/csv_scraper_graph_multi_azure.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Azure/depth_search_graph_azure.md b/docs/Examples/Azure/depth_search_graph_azure.md deleted file mode 100644 index 53e92b33..00000000 --- a/docs/Examples/Azure/depth_search_graph_azure.md +++ /dev/null @@ -1,29 +0,0 @@ -```python -""" -depth_search_graph_azure example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Azure/json_scraper_azure.md b/docs/Examples/Azure/json_scraper_azure.md deleted file mode 100644 index d83162d6..00000000 --- a/docs/Examples/Azure/json_scraper_azure.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Initialize the model instances -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -smart_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Azure/json_scraper_multi_azure.md b/docs/Examples/Azure/json_scraper_multi_azure.md deleted file mode 100644 index a63c5ee1..00000000 --- a/docs/Examples/Azure/json_scraper_multi_azure.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Azure/rate_limit_azure.md b/docs/Examples/Azure/rate_limit_azure.md deleted file mode 100644 index 490fa3b0..00000000 --- a/docs/Examples/Azure/rate_limit_azure.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - - -# required environment variable in .env -# AZURE_OPENAI_ENDPOINT -# AZURE_OPENAI_CHAT_DEPLOYMENT_NAME -# MODEL_NAME -# AZURE_OPENAI_API_KEY -# OPENAI_API_TYPE -# AZURE_OPENAI_API_VERSION -# AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME -load_dotenv() - - -# ************************************************ -# Initialize the model instances -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o", - "rate_limit": { - "requests_per_second": 1 - }, - }, - "verbose": True, - "headless": False -} - -smart_scraper_graph = SmartScraperGraph( - prompt="""List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, - event_end_date, event_end_time, location, event_mode, event_category, - third_party_redirect, no_of_days, - time_in_hours, hosted_or_attending, refreshments_type, - registration_available, registration_link""", - # also accepts a string with the already downloaded HTML code - source="https://www.hmhco.com/event", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Azure/scrape_plain_text_azure.md b/docs/Examples/Azure/scrape_plain_text_azure.md deleted file mode 100644 index 159d1f5a..00000000 --- a/docs/Examples/Azure/scrape_plain_text_azure.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Azure/script_generator_azure.md b/docs/Examples/Azure/script_generator_azure.md deleted file mode 100644 index aa989a0f..00000000 --- a/docs/Examples/Azure/script_generator_azure.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Azure/script_multi_generator_azure.md b/docs/Examples/Azure/script_multi_generator_azure.md deleted file mode 100644 index 582657db..00000000 --- a/docs/Examples/Azure/script_multi_generator_azure.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Azure/search_graph_azure.md b/docs/Examples/Azure/search_graph_azure.md deleted file mode 100644 index c6fdca82..00000000 --- a/docs/Examples/Azure/search_graph_azure.md +++ /dev/null @@ -1,52 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Initialize the model instances -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Azure/search_graph_schema_azure.md b/docs/Examples/Azure/search_graph_schema_azure.md deleted file mode 100644 index 5836a378..00000000 --- a/docs/Examples/Azure/search_graph_schema_azure.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -from pydantic import BaseModel, Field - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Azure/search_link_graph_azure.md b/docs/Examples/Azure/search_link_graph_azure.md deleted file mode 100644 index 6d1079e4..00000000 --- a/docs/Examples/Azure/search_link_graph_azure.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Azure/smart_scraper_azure.md b/docs/Examples/Azure/smart_scraper_azure.md deleted file mode 100644 index 77916d7f..00000000 --- a/docs/Examples/Azure/smart_scraper_azure.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Initialize the model instances -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -smart_scraper_graph = SmartScraperGraph( - prompt="""List me all the events, with the following fields: - company_name, event_name, event_start_date, event_start_time, - event_end_date, event_end_time, location, event_mode, event_category, - third_party_redirect, no_of_days, - time_in_hours, hosted_or_attending, refreshments_type, - registration_available, registration_link""", - # also accepts a string with the already downloaded HTML code - source="https://www.hmhco.com/event", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Azure/smart_scraper_multi_azure.md b/docs/Examples/Azure/smart_scraper_multi_azure.md deleted file mode 100644 index 460a1e50..00000000 --- a/docs/Examples/Azure/smart_scraper_multi_azure.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Azure/smart_scraper_multi_concat_azure.md b/docs/Examples/Azure/smart_scraper_multi_concat_azure.md deleted file mode 100644 index c107b57b..00000000 --- a/docs/Examples/Azure/smart_scraper_multi_concat_azure.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Azure/smart_scraper_schema_azure.md b/docs/Examples/Azure/smart_scraper_schema_azure.md deleted file mode 100644 index 808b3a9b..00000000 --- a/docs/Examples/Azure/smart_scraper_schema_azure.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -import json -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Initialize the model instances -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Azure/xml_scraper_azure.md b/docs/Examples/Azure/xml_scraper_azure.md deleted file mode 100644 index 4d47b801..00000000 --- a/docs/Examples/Azure/xml_scraper_azure.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - - -# ************************************************ -# Initialize the model instances -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -smart_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Azure/xml_scraper_graph_multi_azure.md b/docs/Examples/Azure/xml_scraper_graph_multi_azure.md deleted file mode 100644 index b1e57c7e..00000000 --- a/docs/Examples/Azure/xml_scraper_graph_multi_azure.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o", - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Bedrock/README.md b/docs/Examples/Bedrock/README.md deleted file mode 100644 index 0248d25e..00000000 --- a/docs/Examples/Bedrock/README.md +++ /dev/null @@ -1,4 +0,0 @@ -```python -This folder contains examples of how to use ScrapeGraphAI with [Amazon Bedrock](https://aws.amazon.com/bedrock/) โ›ฐ๏ธ. The examples show how to extract information from websites and files using a natural language prompt. - -![](scrapegraphai_bedrock.png) \ No newline at end of file diff --git a/docs/Examples/Bedrock/code_generator_graph_bedrock.md b/docs/Examples/Bedrock/code_generator_graph_bedrock.md deleted file mode 100644 index a796c8d5..00000000 --- a/docs/Examples/Bedrock/code_generator_graph_bedrock.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" - -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/Bedrock/csv_scraper_bedrock.md b/docs/Examples/Bedrock/csv_scraper_bedrock.md deleted file mode 100644 index 5cc1452b..00000000 --- a/docs/Examples/Bedrock/csv_scraper_bedrock.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" - -import os -import json - -from dotenv import load_dotenv - -import pandas as pd - -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(json.dumps(result, indent=4)) - diff --git a/docs/Examples/Bedrock/csv_scraper_graph_multi_bedrock.md b/docs/Examples/Bedrock/csv_scraper_graph_multi_bedrock.md deleted file mode 100644 index e75a033b..00000000 --- a/docs/Examples/Bedrock/csv_scraper_graph_multi_bedrock.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" - -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Bedrock/custom_graph_bedrock.md b/docs/Examples/Bedrock/custom_graph_bedrock.md deleted file mode 100644 index 863de975..00000000 --- a/docs/Examples/Bedrock/custom_graph_bedrock.md +++ /dev/null @@ -1,126 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" - -import json - -from dotenv import load_dotenv - -from langchain_aws import BedrockEmbeddings -from scrapegraphai.models import Bedrock -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import ( - FetchNode, - ParseNode, - RAGNode, - GenerateAnswerNode, - RobotsNode -) - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = Bedrock({ - 'model_id': graph_config["llm"]["model"].split("/")[-1], - 'model_kwargs': { - 'temperature': 0.0 - }}) -embedder = BedrockEmbeddings(model_id=graph_config["embeddings"]["model"].split("/")[-1]) - -# Define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) - -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) - -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) - -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "List me all the articles", - "url": "https://perinim.github.io/projects" -}) - -# Get the answer from the result -result = result.get("answer", "No answer found.") -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Bedrock/depth_search_graph_bedrock.md b/docs/Examples/Bedrock/depth_search_graph_bedrock.md deleted file mode 100644 index fbde1a40..00000000 --- a/docs/Examples/Bedrock/depth_search_graph_bedrock.md +++ /dev/null @@ -1,26 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -from scrapegraphai.graphs import DepthSearchGraph - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Bedrock/json_scraper_bedrock.md b/docs/Examples/Bedrock/json_scraper_bedrock.md deleted file mode 100644 index d0454db9..00000000 --- a/docs/Examples/Bedrock/json_scraper_bedrock.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all questions and options in the math section, no answers.", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Bedrock/json_scraper_multi_bedrock.md b/docs/Examples/Bedrock/json_scraper_multi_bedrock.md deleted file mode 100644 index 181ec760..00000000 --- a/docs/Examples/Bedrock/json_scraper_multi_bedrock.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from scrapegraphai.graphs import JSONScraperMultiGraph - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Bedrock/rate_limit_bedrock.md b/docs/Examples/Bedrock/rate_limit_bedrock.md deleted file mode 100644 index 19b9583f..00000000 --- a/docs/Examples/Bedrock/rate_limit_bedrock.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0, - "rate_limit": { - "requests_per_second": 1 - }, - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Bedrock/scrape_plain_text_bedrock.md b/docs/Examples/Bedrock/scrape_plain_text_bedrock.md deleted file mode 100644 index e7de926a..00000000 --- a/docs/Examples/Bedrock/scrape_plain_text_bedrock.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Bedrock/script_generator_bedrock.md b/docs/Examples/Bedrock/script_generator_bedrock.md deleted file mode 100644 index d1be9d5d..00000000 --- a/docs/Examples/Bedrock/script_generator_bedrock.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Bedrock/script_multi_generator_bedrock.md b/docs/Examples/Bedrock/script_multi_generator_bedrock.md deleted file mode 100644 index 88334022..00000000 --- a/docs/Examples/Bedrock/script_multi_generator_bedrock.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Bedrock/search_graph_bedrock.md b/docs/Examples/Bedrock/search_graph_bedrock.md deleted file mode 100644 index d9edf487..00000000 --- a/docs/Examples/Bedrock/search_graph_bedrock.md +++ /dev/null @@ -1,28 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Bedrock/search_graph_schema_bedrock.md b/docs/Examples/Bedrock/search_graph_schema_bedrock.md deleted file mode 100644 index 9d7227bd..00000000 --- a/docs/Examples/Bedrock/search_graph_schema_bedrock.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Example of Search Graph -""" -from typing import List -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Bedrock/search_link_graph_bedrock.md b/docs/Examples/Bedrock/search_link_graph_bedrock.md deleted file mode 100644 index 1d80784b..00000000 --- a/docs/Examples/Bedrock/search_link_graph_bedrock.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Bedrock/smart_scraper_bedrock.md b/docs/Examples/Bedrock/smart_scraper_bedrock.md deleted file mode 100644 index e38c7dd4..00000000 --- a/docs/Examples/Bedrock/smart_scraper_bedrock.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Bedrock/smart_scraper_multi_bedrock.md b/docs/Examples/Bedrock/smart_scraper_multi_bedrock.md deleted file mode 100644 index da51fb7d..00000000 --- a/docs/Examples/Bedrock/smart_scraper_multi_bedrock.md +++ /dev/null @@ -1,35 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import json -from scrapegraphai.graphs import SmartScraperMultiGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Bedrock/smart_scraper_multi_concat_bedrock.md b/docs/Examples/Bedrock/smart_scraper_multi_concat_bedrock.md deleted file mode 100644 index 36bb96df..00000000 --- a/docs/Examples/Bedrock/smart_scraper_multi_concat_bedrock.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import json -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Bedrock/smart_scraper_schema_bedrock.md b/docs/Examples/Bedrock/smart_scraper_schema_bedrock.md deleted file mode 100644 index c7319fc6..00000000 --- a/docs/Examples/Bedrock/smart_scraper_schema_bedrock.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from typing import List -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Bedrock/xml_scraper_bedrock.md b/docs/Examples/Bedrock/xml_scraper_bedrock.md deleted file mode 100644 index e04b6de2..00000000 --- a/docs/Examples/Bedrock/xml_scraper_bedrock.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - }, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books. Skip the preamble.", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Bedrock/xml_scraper_graph_multi_bedrock.md b/docs/Examples/Bedrock/xml_scraper_graph_multi_bedrock.md deleted file mode 100644 index f742c9f5..00000000 --- a/docs/Examples/Bedrock/xml_scraper_graph_multi_bedrock.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - }, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Deepseek/code_generator_graph_deepseek.md b/docs/Examples/Deepseek/code_generator_graph_deepseek.md deleted file mode 100644 index 7ba35090..00000000 --- a/docs/Examples/Deepseek/code_generator_graph_deepseek.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Deepseek/csv_scraper_deepseek.md b/docs/Examples/Deepseek/csv_scraper_deepseek.md deleted file mode 100644 index 49273d1d..00000000 --- a/docs/Examples/Deepseek/csv_scraper_deepseek.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Deepseek/csv_scraper_graph_multi_deepseek.md b/docs/Examples/Deepseek/csv_scraper_graph_multi_deepseek.md deleted file mode 100644 index 76dac83d..00000000 --- a/docs/Examples/Deepseek/csv_scraper_graph_multi_deepseek.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Deepseek/depth_search_graph_deepseek.md b/docs/Examples/Deepseek/depth_search_graph_deepseek.md deleted file mode 100644 index 6799e628..00000000 --- a/docs/Examples/Deepseek/depth_search_graph_deepseek.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Deepseek/json_scraper_deepseek.md b/docs/Examples/Deepseek/json_scraper_deepseek.md deleted file mode 100644 index 727d596f..00000000 --- a/docs/Examples/Deepseek/json_scraper_deepseek.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/Deepseek/json_scraper_multi_deepseek.md b/docs/Examples/Deepseek/json_scraper_multi_deepseek.md deleted file mode 100644 index 52db797e..00000000 --- a/docs/Examples/Deepseek/json_scraper_multi_deepseek.md +++ /dev/null @@ -1,38 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Deepseek/rate_limit_deepseek.md b/docs/Examples/Deepseek/rate_limit_deepseek.md deleted file mode 100644 index 5c9e937c..00000000 --- a/docs/Examples/Deepseek/rate_limit_deepseek.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - "rate_limit": { - "requests_per_second": 1 - } - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Deepseek/scrape_plain_text_deepseek.md b/docs/Examples/Deepseek/scrape_plain_text_deepseek.md deleted file mode 100644 index 85381291..00000000 --- a/docs/Examples/Deepseek/scrape_plain_text_deepseek.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Deepseek/script_generator_deepseek.md b/docs/Examples/Deepseek/script_generator_deepseek.md deleted file mode 100644 index 90fee8ed..00000000 --- a/docs/Examples/Deepseek/script_generator_deepseek.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Deepseek/script_multi_generator_deepseek.md b/docs/Examples/Deepseek/script_multi_generator_deepseek.md deleted file mode 100644 index 604c8878..00000000 --- a/docs/Examples/Deepseek/script_multi_generator_deepseek.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Deepseek/search_graph_deepseek.md b/docs/Examples/Deepseek/search_graph_deepseek.md deleted file mode 100644 index 866b10fb..00000000 --- a/docs/Examples/Deepseek/search_graph_deepseek.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Deepseek/search_graph_schema_deepseek.md b/docs/Examples/Deepseek/search_graph_schema_deepseek.md deleted file mode 100644 index cf481308..00000000 --- a/docs/Examples/Deepseek/search_graph_schema_deepseek.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Deepseek/search_link_graph_deepseek.md b/docs/Examples/Deepseek/search_link_graph_deepseek.md deleted file mode 100644 index a5dcb1e9..00000000 --- a/docs/Examples/Deepseek/search_link_graph_deepseek.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Deepseek/smart_scraper_deepseek.md b/docs/Examples/Deepseek/smart_scraper_deepseek.md deleted file mode 100644 index e88ce75d..00000000 --- a/docs/Examples/Deepseek/smart_scraper_deepseek.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Deepseek/smart_scraper_multi_concat_deepseek.md b/docs/Examples/Deepseek/smart_scraper_multi_concat_deepseek.md deleted file mode 100644 index ce1ffa4c..00000000 --- a/docs/Examples/Deepseek/smart_scraper_multi_concat_deepseek.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Deepseek/smart_scraper_multi_deepseek.md b/docs/Examples/Deepseek/smart_scraper_multi_deepseek.md deleted file mode 100644 index 9caeb8c2..00000000 --- a/docs/Examples/Deepseek/smart_scraper_multi_deepseek.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Deepseek/smart_scraper_schema_deepseek.md b/docs/Examples/Deepseek/smart_scraper_schema_deepseek.md deleted file mode 100644 index 8a1735ed..00000000 --- a/docs/Examples/Deepseek/smart_scraper_schema_deepseek.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Deepseek/xml_scraper_deepseek.md b/docs/Examples/Deepseek/xml_scraper_deepseek.md deleted file mode 100644 index 628e0789..00000000 --- a/docs/Examples/Deepseek/xml_scraper_deepseek.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Deepseek/xml_scraper_graph_multi_deepseek.md b/docs/Examples/Deepseek/xml_scraper_graph_multi_deepseek.md deleted file mode 100644 index 013086a3..00000000 --- a/docs/Examples/Deepseek/xml_scraper_graph_multi_deepseek.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Ernie/code_generator_graph_ernie.md b/docs/Examples/Ernie/code_generator_graph_ernie.md deleted file mode 100644 index 623ce5d0..00000000 --- a/docs/Examples/Ernie/code_generator_graph_ernie.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/Ernie/csv_scraper_ernie.md b/docs/Examples/Ernie/csv_scraper_ernie.md deleted file mode 100644 index f893bf6a..00000000 --- a/docs/Examples/Ernie/csv_scraper_ernie.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - } -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Ernie/custom_graph_ernie.md b/docs/Examples/Ernie/custom_graph_ernie.md deleted file mode 100644 index 6679d32a..00000000 --- a/docs/Examples/Ernie/custom_graph_ernie.md +++ /dev/null @@ -1,107 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -from langchain_openai import OpenAIEmbeddings -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - } -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/Ernie/depth_search_graph_ernie.md b/docs/Examples/Ernie/depth_search_graph_ernie.md deleted file mode 100644 index 48ce19ef..00000000 --- a/docs/Examples/Ernie/depth_search_graph_ernie.md +++ /dev/null @@ -1,27 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -from scrapegraphai.graphs import DepthSearchGraph - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Ernie/json_scraper_ernie.md b/docs/Examples/Ernie/json_scraper_ernie.md deleted file mode 100644 index 56306eed..00000000 --- a/docs/Examples/Ernie/json_scraper_ernie.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - } -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/Ernie/rate_limit_ernie.md b/docs/Examples/Ernie/rate_limit_ernie.md deleted file mode 100644 index a19ad3dd..00000000 --- a/docs/Examples/Ernie/rate_limit_ernie.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1, - "rate_limit": { - "requests_per_second": 1 - }, - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Ernie/scrape_plain_text_ernie.md b/docs/Examples/Ernie/scrape_plain_text_ernie.md deleted file mode 100644 index 6733adff..00000000 --- a/docs/Examples/Ernie/scrape_plain_text_ernie.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Ernie/script_generator_ernie.md b/docs/Examples/Ernie/script_generator_ernie.md deleted file mode 100644 index 4f729fe6..00000000 --- a/docs/Examples/Ernie/script_generator_ernie.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Ernie/script_multi_generator_ernie.md b/docs/Examples/Ernie/script_multi_generator_ernie.md deleted file mode 100644 index 235be684..00000000 --- a/docs/Examples/Ernie/script_multi_generator_ernie.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Ernie/search_graph_ernie.md b/docs/Examples/Ernie/search_graph_ernie.md deleted file mode 100644 index 52add57e..00000000 --- a/docs/Examples/Ernie/search_graph_ernie.md +++ /dev/null @@ -1,34 +0,0 @@ -```python -""" -Example of Search Graph -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Ernie/search_link_graph_ernie.md b/docs/Examples/Ernie/search_link_graph_ernie.md deleted file mode 100644 index 2b5b192b..00000000 --- a/docs/Examples/Ernie/search_link_graph_ernie.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Ernie/smart_scraper_ernie.md b/docs/Examples/Ernie/smart_scraper_ernie.md deleted file mode 100644 index bf665aa2..00000000 --- a/docs/Examples/Ernie/smart_scraper_ernie.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Ernie/smart_scraper_multi_concat_ernie.md b/docs/Examples/Ernie/smart_scraper_multi_concat_ernie.md deleted file mode 100644 index bc8310af..00000000 --- a/docs/Examples/Ernie/smart_scraper_multi_concat_ernie.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import json -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "library": "beautifulsoup" -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Ernie/smart_scraper_multi_ernie.md b/docs/Examples/Ernie/smart_scraper_multi_ernie.md deleted file mode 100644 index 3071638d..00000000 --- a/docs/Examples/Ernie/smart_scraper_multi_ernie.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Ernie/smart_scraper_schema_ernie.md b/docs/Examples/Ernie/smart_scraper_schema_ernie.md deleted file mode 100644 index ef2c0249..00000000 --- a/docs/Examples/Ernie/smart_scraper_schema_ernie.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import json -import os -from typing import Dict -from dotenv import load_dotenv -from pydantic import BaseModel -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ -class Project(BaseModel): - title: str - description: str - - -class Projects(BaseModel): - Projects: Dict[str, Project] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config, -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Ernie/speech_graph_ernie.md b/docs/Examples/Ernie/speech_graph_ernie.md deleted file mode 100644 index 2a85cb95..00000000 --- a/docs/Examples/Ernie/speech_graph_ernie.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SpeechSummaryGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SpeechGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define audio output path -# ************************************************ - -FILE_NAME = "website_summary.mp3" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -output_path = os.path.join(curr_dir, FILE_NAME) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "tts_model": { - "api_key": openai_key, - "model": "tts-1", - "voice": "alloy" - }, - "output_path": output_path, -} - -# ************************************************ -# Create the SpeechGraph instance and run it -# ************************************************ - -speech_graph = SpeechGraph( - prompt="Make a detailed audio summary of the projects.", - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = speech_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = speech_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Ernie/xml_scraper_ernie.md b/docs/Examples/Ernie/xml_scraper_ernie.md deleted file mode 100644 index 393eb6da..00000000 --- a/docs/Examples/Ernie/xml_scraper_ernie.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "verbose":False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/docs/Examples/Groq/code_generator_graph_groq.md b/docs/Examples/Groq/code_generator_graph_groq.md deleted file mode 100644 index f5df87c0..00000000 --- a/docs/Examples/Groq/code_generator_graph_groq.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Groq/csv_scraper_graph_multi_groq.md b/docs/Examples/Groq/csv_scraper_graph_multi_groq.md deleted file mode 100644 index bde981e7..00000000 --- a/docs/Examples/Groq/csv_scraper_graph_multi_groq.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Groq/csv_scraper_groq.md b/docs/Examples/Groq/csv_scraper_groq.md deleted file mode 100644 index a0ade8bd..00000000 --- a/docs/Examples/Groq/csv_scraper_groq.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, -} -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Groq/custom_graph_groq.md b/docs/Examples/Groq/custom_graph_groq.md deleted file mode 100644 index 1c4fecd6..00000000 --- a/docs/Examples/Groq/custom_graph_groq.md +++ /dev/null @@ -1,100 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -import os -from dotenv import load_dotenv -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) - -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/Groq/depth_search_graph_groq.md b/docs/Examples/Groq/depth_search_graph_groq.md deleted file mode 100644 index 9e116d81..00000000 --- a/docs/Examples/Groq/depth_search_graph_groq.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Groq/json_scraper_groq.md b/docs/Examples/Groq/json_scraper_groq.md deleted file mode 100644 index fae91fbf..00000000 --- a/docs/Examples/Groq/json_scraper_groq.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/Groq/json_scraper_multi_groq.md b/docs/Examples/Groq/json_scraper_multi_groq.md deleted file mode 100644 index cc66770e..00000000 --- a/docs/Examples/Groq/json_scraper_multi_groq.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "library": "beautifulsoup" -} -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Groq/rate_limit_groq.md b/docs/Examples/Groq/rate_limit_groq.md deleted file mode 100644 index 7f50fbef..00000000 --- a/docs/Examples/Groq/rate_limit_groq.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0, - "rate_limit": { - "requests_per_second": 1 - } - }, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Groq/scrape_plain_text_groq.md b/docs/Examples/Groq/scrape_plain_text_groq.md deleted file mode 100644 index 664f816c..00000000 --- a/docs/Examples/Groq/scrape_plain_text_groq.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Groq/script_generator_groq.md b/docs/Examples/Groq/script_generator_groq.md deleted file mode 100644 index a52d69bd..00000000 --- a/docs/Examples/Groq/script_generator_groq.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "library": "beautifulsoup" -} -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Groq/script_multi_generator_groq.md b/docs/Examples/Groq/script_multi_generator_groq.md deleted file mode 100644 index 87c0d4ef..00000000 --- a/docs/Examples/Groq/script_multi_generator_groq.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Groq/search_graph_groq.md b/docs/Examples/Groq/search_graph_groq.md deleted file mode 100644 index ca327517..00000000 --- a/docs/Examples/Groq/search_graph_groq.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Groq/search_graph_schema_groq.md b/docs/Examples/Groq/search_graph_schema_groq.md deleted file mode 100644 index 653b6230..00000000 --- a/docs/Examples/Groq/search_graph_schema_groq.md +++ /dev/null @@ -1,63 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Groq/search_link_graph_groq.md b/docs/Examples/Groq/search_link_graph_groq.md deleted file mode 100644 index 5aafc92f..00000000 --- a/docs/Examples/Groq/search_link_graph_groq.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Groq/smart_scraper_groq.md b/docs/Examples/Groq/smart_scraper_groq.md deleted file mode 100644 index 66f4d011..00000000 --- a/docs/Examples/Groq/smart_scraper_groq.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Groq/smart_scraper_multi_concat_groq.md b/docs/Examples/Groq/smart_scraper_multi_concat_groq.md deleted file mode 100644 index 003dd845..00000000 --- a/docs/Examples/Groq/smart_scraper_multi_concat_groq.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Groq/smart_scraper_multi_groq.md b/docs/Examples/Groq/smart_scraper_multi_groq.md deleted file mode 100644 index 8dc1644a..00000000 --- a/docs/Examples/Groq/smart_scraper_multi_groq.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False -} -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Groq/smart_scraper_schema_groq.md b/docs/Examples/Groq/smart_scraper_schema_groq.md deleted file mode 100644 index a372dc64..00000000 --- a/docs/Examples/Groq/smart_scraper_schema_groq.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Groq/xml_scraper_graph_multi_groq.md b/docs/Examples/Groq/xml_scraper_graph_multi_groq.md deleted file mode 100644 index 25d67453..00000000 --- a/docs/Examples/Groq/xml_scraper_graph_multi_groq.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Groq/xml_scraper_groq.md b/docs/Examples/Groq/xml_scraper_groq.md deleted file mode 100644 index 49e692be..00000000 --- a/docs/Examples/Groq/xml_scraper_groq.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False -} -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/docs/Examples/Mistral/code_generator_graph_mistral.md b/docs/Examples/Mistral/code_generator_graph_mistral.md deleted file mode 100644 index 792a5302..00000000 --- a/docs/Examples/Mistral/code_generator_graph_mistral.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Mistral/csv_scraper_graph_multi_mistral.md b/docs/Examples/Mistral/csv_scraper_graph_multi_mistral.md deleted file mode 100644 index 45016459..00000000 --- a/docs/Examples/Mistral/csv_scraper_graph_multi_mistral.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Mistral/csv_scraper_mistral.md b/docs/Examples/Mistral/csv_scraper_mistral.md deleted file mode 100644 index c6309743..00000000 --- a/docs/Examples/Mistral/csv_scraper_mistral.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Mistral/custom_graph_mistral.md b/docs/Examples/Mistral/custom_graph_mistral.md deleted file mode 100644 index 3d566343..00000000 --- a/docs/Examples/Mistral/custom_graph_mistral.md +++ /dev/null @@ -1,109 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -import os -from dotenv import load_dotenv -from langchain_mistralai import ChatMistralAI, MistralAIEmbeddings -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatMistralAI(**graph_config["llm"]) -embedder = MistralAIEmbeddings(api_key=llm_model.mistral_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/Mistral/depth_search_graph_mistral.md b/docs/Examples/Mistral/depth_search_graph_mistral.md deleted file mode 100644 index 9855215f..00000000 --- a/docs/Examples/Mistral/depth_search_graph_mistral.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Mistral/json_scraper_mistral.md b/docs/Examples/Mistral/json_scraper_mistral.md deleted file mode 100644 index 74b61d25..00000000 --- a/docs/Examples/Mistral/json_scraper_mistral.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/Mistral/json_scraper_multi_mistral.md b/docs/Examples/Mistral/json_scraper_multi_mistral.md deleted file mode 100644 index 54a4bd2e..00000000 --- a/docs/Examples/Mistral/json_scraper_multi_mistral.md +++ /dev/null @@ -1,38 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - } -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Mistral/md_scraper_mistral.md b/docs/Examples/Mistral/md_scraper_mistral.md deleted file mode 100644 index 67a2a5bf..00000000 --- a/docs/Examples/Mistral/md_scraper_mistral.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using DocumentScraperGraph from MD documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the MD file -# ************************************************ - -FILE_NAME = "inputs/markdown_example.md" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -# ************************************************ -# Create the DocumentScraperGraph instance and run it -# ************************************************ - -md_scraper_graph = DocumentScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = md_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = md_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Mistral/rate_limit_mistral.md b/docs/Examples/Mistral/rate_limit_mistral.md deleted file mode 100644 index 3a930d61..00000000 --- a/docs/Examples/Mistral/rate_limit_mistral.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -import json -from scrapegraphai.graphs import SmartScraperGraph -from dotenv import load_dotenv - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("MISTRAL_API_KEY"), - "model": "mistralai/open-mistral-nemo", - "rate_limit": { - "requests_per_second": 1 - } - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Mistral/readme.md b/docs/Examples/Mistral/readme.md deleted file mode 100644 index daa88f49..00000000 --- a/docs/Examples/Mistral/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -```python -This folder contains examples of how to use ScrapeGraph-AI with Mistral, an LLM provider. The examples show how to extract information from a website using a natural language prompt. \ No newline at end of file diff --git a/docs/Examples/Mistral/scrape_plain_text_mistral.md b/docs/Examples/Mistral/scrape_plain_text_mistral.md deleted file mode 100644 index f85b9bb5..00000000 --- a/docs/Examples/Mistral/scrape_plain_text_mistral.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Mistral/script_generator_mistral.md b/docs/Examples/Mistral/script_generator_mistral.md deleted file mode 100644 index 25076407..00000000 --- a/docs/Examples/Mistral/script_generator_mistral.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Mistral/script_generator_schema_mistral.md b/docs/Examples/Mistral/script_generator_schema_mistral.md deleted file mode 100644 index d2dcfefc..00000000 --- a/docs/Examples/Mistral/script_generator_schema_mistral.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info -from pydantic import BaseModel, Field -from typing import List - -load_dotenv() - -# ************************************************ -# Define the schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config, - schema=Projects -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Mistral/script_multi_generator_mistral.md b/docs/Examples/Mistral/script_multi_generator_mistral.md deleted file mode 100644 index 23f070e6..00000000 --- a/docs/Examples/Mistral/script_multi_generator_mistral.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Mistral/search_graph_mistral.md b/docs/Examples/Mistral/search_graph_mistral.md deleted file mode 100644 index e651452e..00000000 --- a/docs/Examples/Mistral/search_graph_mistral.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Mistral/search_graph_schema_mistral.md b/docs/Examples/Mistral/search_graph_schema_mistral.md deleted file mode 100644 index 3ceb16e2..00000000 --- a/docs/Examples/Mistral/search_graph_schema_mistral.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Mistral/search_link_graph_mistral.md b/docs/Examples/Mistral/search_link_graph_mistral.md deleted file mode 100644 index 46f108c0..00000000 --- a/docs/Examples/Mistral/search_link_graph_mistral.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchLinkGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SearchLinkGraph instance and run it -# ************************************************ - -smart_scraper_graph = SearchLinkGraph( - source="https://sport.sky.it/nba?gr=www", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Mistral/smart_scraper_mistral.md b/docs/Examples/Mistral/smart_scraper_mistral.md deleted file mode 100644 index 047d98cf..00000000 --- a/docs/Examples/Mistral/smart_scraper_mistral.md +++ /dev/null @@ -1,37 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("MISTRAL_API_KEY"), - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Mistral/smart_scraper_multi_concat_mistral.md b/docs/Examples/Mistral/smart_scraper_multi_concat_mistral.md deleted file mode 100644 index 124eb357..00000000 --- a/docs/Examples/Mistral/smart_scraper_multi_concat_mistral.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "api_key": os.getenv("MISTRAL_API_KEY"), - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Mistral/smart_scraper_multi_mistral.md b/docs/Examples/Mistral/smart_scraper_multi_mistral.md deleted file mode 100644 index ec2c68a0..00000000 --- a/docs/Examples/Mistral/smart_scraper_multi_mistral.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Mistral/smart_scraper_schema_mistral.md b/docs/Examples/Mistral/smart_scraper_schema_mistral.md deleted file mode 100644 index 98128fcf..00000000 --- a/docs/Examples/Mistral/smart_scraper_schema_mistral.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key":mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) diff --git a/docs/Examples/Mistral/xml_scraper_graph_multi_mistral.md b/docs/Examples/Mistral/xml_scraper_graph_multi_mistral.md deleted file mode 100644 index cd76879a..00000000 --- a/docs/Examples/Mistral/xml_scraper_graph_multi_mistral.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key":mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Mistral/xml_scraper_mistral.md b/docs/Examples/Mistral/xml_scraper_mistral.md deleted file mode 100644 index d52bd314..00000000 --- a/docs/Examples/Mistral/xml_scraper_mistral.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, - "verbose":False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/code_generator_graph_nemotron.md b/docs/Examples/Nemotron/code_generator_graph_nemotron.md deleted file mode 100644 index a0130289..00000000 --- a/docs/Examples/Nemotron/code_generator_graph_nemotron.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_APIKEY"), - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Nemotron/csv_scraper_graph_multi_nemotron.md b/docs/Examples/Nemotron/csv_scraper_graph_multi_nemotron.md deleted file mode 100644 index 51ebcd94..00000000 --- a/docs/Examples/Nemotron/csv_scraper_graph_multi_nemotron.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -import pandas as pd -from dotenv import load_dotenv -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_APIKEY"), - "model": "nvidia/meta/llama3-70b-instruct", - } -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/csv_scraper_nemotron.md b/docs/Examples/Nemotron/csv_scraper_nemotron.md deleted file mode 100644 index 5b8a4876..00000000 --- a/docs/Examples/Nemotron/csv_scraper_nemotron.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/depth_search_graph_nemotron.md b/docs/Examples/Nemotron/depth_search_graph_nemotron.md deleted file mode 100644 index ed90e987..00000000 --- a/docs/Examples/Nemotron/depth_search_graph_nemotron.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_KEY"), - "model": "claude-3-haiku-20240307", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Nemotron/json_scraper_multi_nemotron.md b/docs/Examples/Nemotron/json_scraper_multi_nemotron.md deleted file mode 100644 index 5c475623..00000000 --- a/docs/Examples/Nemotron/json_scraper_multi_nemotron.md +++ /dev/null @@ -1,38 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Nemotron/json_scraper_nemotron.md b/docs/Examples/Nemotron/json_scraper_nemotron.md deleted file mode 100644 index 0c78ff89..00000000 --- a/docs/Examples/Nemotron/json_scraper_nemotron.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/md_scraper_nemotron.md b/docs/Examples/Nemotron/md_scraper_nemotron.md deleted file mode 100644 index a5802592..00000000 --- a/docs/Examples/Nemotron/md_scraper_nemotron.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using DocumentScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/markdown_example.md" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the DocumentScraperGraph instance and run it -# ************************************************ - -md_scraper_graph = DocumentScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = md_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = md_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/rate_limit_nemotron.md b/docs/Examples/Nemotron/rate_limit_nemotron.md deleted file mode 100644 index ff549deb..00000000 --- a/docs/Examples/Nemotron/rate_limit_nemotron.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_KEY"), - "model": "nvidia/meta/llama3-70b-instruct", - "rate_limit": { - "requests_per_second": 1 - } - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="Extract me the python code inside the page", - source="https://www.exploit-db.com/exploits/51447", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/scrape_plain_text_nemotron.md b/docs/Examples/Nemotron/scrape_plain_text_nemotron.md deleted file mode 100644 index d197c6d1..00000000 --- a/docs/Examples/Nemotron/scrape_plain_text_nemotron.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/script_generator_nemotron.md b/docs/Examples/Nemotron/script_generator_nemotron.md deleted file mode 100644 index dfaa956a..00000000 --- a/docs/Examples/Nemotron/script_generator_nemotron.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/script_generator_schema_nemotron.md b/docs/Examples/Nemotron/script_generator_schema_nemotron.md deleted file mode 100644 index f82d26da..00000000 --- a/docs/Examples/Nemotron/script_generator_schema_nemotron.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from typing import List -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config, - schema=Projects -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/script_multi_generator_nemotron.md b/docs/Examples/Nemotron/script_multi_generator_nemotron.md deleted file mode 100644 index 0bdd467a..00000000 --- a/docs/Examples/Nemotron/script_multi_generator_nemotron.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/search_graph_nemotron.md b/docs/Examples/Nemotron/search_graph_nemotron.md deleted file mode 100644 index 28b33621..00000000 --- a/docs/Examples/Nemotron/search_graph_nemotron.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Nemotron/search_graph_schema_nemotron.md b/docs/Examples/Nemotron/search_graph_schema_nemotron.md deleted file mode 100644 index a0532381..00000000 --- a/docs/Examples/Nemotron/search_graph_schema_nemotron.md +++ /dev/null @@ -1,63 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/search_link_graph_nemotron.md b/docs/Examples/Nemotron/search_link_graph_nemotron.md deleted file mode 100644 index 70dfa4fb..00000000 --- a/docs/Examples/Nemotron/search_link_graph_nemotron.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchLinkGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_APIKEY"), - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SearchLinkGraph instance and run it -# ************************************************ - -smart_scraper_graph = SearchLinkGraph( - source="https://sport.sky.it/nba?gr=www", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/smart_scraper_multi_concat_nemotron.md b/docs/Examples/Nemotron/smart_scraper_multi_concat_nemotron.md deleted file mode 100644 index 09fb055d..00000000 --- a/docs/Examples/Nemotron/smart_scraper_multi_concat_nemotron.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_APIKEY"), - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Nemotron/smart_scraper_multi_nemotron.md b/docs/Examples/Nemotron/smart_scraper_multi_nemotron.md deleted file mode 100644 index 5c7393b7..00000000 --- a/docs/Examples/Nemotron/smart_scraper_multi_nemotron.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Nemotron/smart_scraper_nemotron.md b/docs/Examples/Nemotron/smart_scraper_nemotron.md deleted file mode 100644 index 2096f9d1..00000000 --- a/docs/Examples/Nemotron/smart_scraper_nemotron.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_KEY"), - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="Extract me the python code inside the page", - source="https://www.exploit-db.com/exploits/51447", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/smart_scraper_schema_nemotron.md b/docs/Examples/Nemotron/smart_scraper_schema_nemotron.md deleted file mode 100644 index 9f471e92..00000000 --- a/docs/Examples/Nemotron/smart_scraper_schema_nemotron.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key":nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) diff --git a/docs/Examples/Nemotron/speech_graph_nemotron.md b/docs/Examples/Nemotron/speech_graph_nemotron.md deleted file mode 100644 index 05c0b157..00000000 --- a/docs/Examples/Nemotron/speech_graph_nemotron.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SpeechSummaryGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SpeechGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define audio output path -# ************************************************ - -FILE_NAME = "website_summary.mp3" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -output_path = os.path.join(curr_dir, FILE_NAME) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -# ************************************************ -# Create the SpeechGraph instance and run it -# ************************************************ - -speech_graph = SpeechGraph( - prompt="Make a detailed audio summary of the projects.", - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = speech_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = speech_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Nemotron/xml_scraper_graph_nemotron.md b/docs/Examples/Nemotron/xml_scraper_graph_nemotron.md deleted file mode 100644 index 76ff63e3..00000000 --- a/docs/Examples/Nemotron/xml_scraper_graph_nemotron.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" - -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key":nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Nemotron/xml_scraper_nemotron.md b/docs/Examples/Nemotron/xml_scraper_nemotron.md deleted file mode 100644 index ecbd8bb1..00000000 --- a/docs/Examples/Nemotron/xml_scraper_nemotron.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, - "verbose":False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/docs/Examples/Oneapi/code_generator_graph_oneapi.md b/docs/Examples/Oneapi/code_generator_graph_oneapi.md deleted file mode 100644 index 937c72ca..00000000 --- a/docs/Examples/Oneapi/code_generator_graph_oneapi.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/Oneapi/csv_scraper_graph_multi_oneapi.md b/docs/Examples/Oneapi/csv_scraper_graph_multi_oneapi.md deleted file mode 100644 index 5ad816ea..00000000 --- a/docs/Examples/Oneapi/csv_scraper_graph_multi_oneapi.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Oneapi/csv_scraper_oneapi.md b/docs/Examples/Oneapi/csv_scraper_oneapi.md deleted file mode 100644 index 0ecfada2..00000000 --- a/docs/Examples/Oneapi/csv_scraper_oneapi.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Oneapi/custom_graph_oneapi.md b/docs/Examples/Oneapi/custom_graph_oneapi.md deleted file mode 100644 index 02f29cef..00000000 --- a/docs/Examples/Oneapi/custom_graph_oneapi.md +++ /dev/null @@ -1,106 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -from langchain_openai import OpenAIEmbeddings -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/Oneapi/depth_search_graph_onenapi.md b/docs/Examples/Oneapi/depth_search_graph_onenapi.md deleted file mode 100644 index e1291d55..00000000 --- a/docs/Examples/Oneapi/depth_search_graph_onenapi.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Oneapi/json_scraper_multi_oneapi.md b/docs/Examples/Oneapi/json_scraper_multi_oneapi.md deleted file mode 100644 index 7e6f6425..00000000 --- a/docs/Examples/Oneapi/json_scraper_multi_oneapi.md +++ /dev/null @@ -1,34 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from scrapegraphai.graphs import JSONScraperMultiGraph - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Oneapi/json_scraper_oneapi.md b/docs/Examples/Oneapi/json_scraper_oneapi.md deleted file mode 100644 index 7dcfc48b..00000000 --- a/docs/Examples/Oneapi/json_scraper_oneapi.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Oneapi/rate_limit_oneapi.md b/docs/Examples/Oneapi/rate_limit_oneapi.md deleted file mode 100644 index 438e9f47..00000000 --- a/docs/Examples/Oneapi/rate_limit_oneapi.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - "rate_limit": { - "requests_per_second": 1 - } - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the titles", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Oneapi/scrape_plain_text_oneapi.md b/docs/Examples/Oneapi/scrape_plain_text_oneapi.md deleted file mode 100644 index f49a45ec..00000000 --- a/docs/Examples/Oneapi/scrape_plain_text_oneapi.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Oneapi/script_generator_oneapi.md b/docs/Examples/Oneapi/script_generator_oneapi.md deleted file mode 100644 index f924be01..00000000 --- a/docs/Examples/Oneapi/script_generator_oneapi.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Oneapi/script_multi_generator_oneapi.md b/docs/Examples/Oneapi/script_multi_generator_oneapi.md deleted file mode 100644 index 8b2deefd..00000000 --- a/docs/Examples/Oneapi/script_multi_generator_oneapi.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Oneapi/search_graph_oneapi.md b/docs/Examples/Oneapi/search_graph_oneapi.md deleted file mode 100644 index 9e3f12c1..00000000 --- a/docs/Examples/Oneapi/search_graph_oneapi.md +++ /dev/null @@ -1,30 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Oneapi/search_graph_schema_oneapi.md b/docs/Examples/Oneapi/search_graph_schema_oneapi.md deleted file mode 100644 index bca60ffe..00000000 --- a/docs/Examples/Oneapi/search_graph_schema_oneapi.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -from pydantic import BaseModel, Field -from typing import List - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Oneapi/smart_scraper_multi_concat_oneapi.md b/docs/Examples/Oneapi/smart_scraper_multi_concat_oneapi.md deleted file mode 100644 index bf165c55..00000000 --- a/docs/Examples/Oneapi/smart_scraper_multi_concat_oneapi.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import json -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Oneapi/smart_scraper_multi_oneapi.md b/docs/Examples/Oneapi/smart_scraper_multi_oneapi.md deleted file mode 100644 index d2f3e0be..00000000 --- a/docs/Examples/Oneapi/smart_scraper_multi_oneapi.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Oneapi/smart_scraper_oneapi.md b/docs/Examples/Oneapi/smart_scraper_oneapi.md deleted file mode 100644 index 62abcd54..00000000 --- a/docs/Examples/Oneapi/smart_scraper_oneapi.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the titles", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Oneapi/smart_scraper_schema_oneapi.md b/docs/Examples/Oneapi/smart_scraper_schema_oneapi.md deleted file mode 100644 index 694ce965..00000000 --- a/docs/Examples/Oneapi/smart_scraper_schema_oneapi.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper and OneAPI -""" -from typing import List -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ********************************************* - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config, - schema=Projects -) - -# ************************************************ -# Get graph execution info -# ************************************************ -result = smart_scraper_graph.run() -print(result) -print(prettify_exec_info(result)) diff --git a/docs/Examples/Oneapi/smartscraper_oneapi.md b/docs/Examples/Oneapi/smartscraper_oneapi.md deleted file mode 100644 index 38e35a2a..00000000 --- a/docs/Examples/Oneapi/smartscraper_oneapi.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ********************************************* - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="่ฏฅ็ฝ‘็ซ™ไธบXXXXX,่ฏทๆๅ–ๅ‡บๆ ‡้ข˜ใ€ๅ‘ๅธƒๆ—ถ้—ดใ€ๅ‘ๅธƒๆฅๆบไปฅๅŠๅ†…ๅฎนๆ‘˜่ฆ,ๅนถไปฅไธญๆ–‡ๅ›ž็ญ”ใ€‚", - # ไนŸๅฏไปฅไฝฟ็”จๅทฒไธ‹่ฝฝ็š„ HTML ไปฃ็ ็š„ๅญ—็ฌฆไธฒ - source="http://XXXX", - config=graph_config -) - -# ************************************************ -# Get graph execution info -# ************************************************ -result = smart_scraper_graph.run() -print(result) -print(prettify_exec_info(result)) diff --git a/docs/Examples/Oneapi/xml_scraper_graph_multi_oneapi.md b/docs/Examples/Oneapi/xml_scraper_graph_multi_oneapi.md deleted file mode 100644 index a08a1cda..00000000 --- a/docs/Examples/Oneapi/xml_scraper_graph_multi_oneapi.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-3.5-turbo", - }, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Oneapi/xml_scraper_oneapi.md b/docs/Examples/Oneapi/xml_scraper_oneapi.md deleted file mode 100644 index 74680ca2..00000000 --- a/docs/Examples/Oneapi/xml_scraper_oneapi.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("ONEAPI_KEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-3.5-turbo", - }, - "verbose":False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/code_generator_graph_openai.md b/docs/Examples/Openai/code_generator_graph_openai.md deleted file mode 100644 index b69f243b..00000000 --- a/docs/Examples/Openai/code_generator_graph_openai.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key":openai_key, - "model": "openai/gpt-4o-mini", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/Openai/csv_scraper_graph_multi_openai.md b/docs/Examples/Openai/csv_scraper_graph_multi_openai.md deleted file mode 100644 index 632e0711..00000000 --- a/docs/Examples/Openai/csv_scraper_graph_multi_openai.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/csv_scraper_openai.md b/docs/Examples/Openai/csv_scraper_openai.md deleted file mode 100644 index fc6cda54..00000000 --- a/docs/Examples/Openai/csv_scraper_openai.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/custom_graph_openai.md b/docs/Examples/Openai/custom_graph_openai.md deleted file mode 100644 index 2ea44ba2..00000000 --- a/docs/Examples/Openai/custom_graph_openai.md +++ /dev/null @@ -1,110 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -import os -from dotenv import load_dotenv -from langchain_openai import OpenAIEmbeddings -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") -graph_config = { - "llm": { - "api_key": openai_key, - "model": "gpt-4o", - }, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/Openai/depth_search_graph_openai.md b/docs/Examples/Openai/depth_search_graph_openai.md deleted file mode 100644 index 59e578aa..00000000 --- a/docs/Examples/Openai/depth_search_graph_openai.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o-mini", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Openai/json_scraper_multi_openai.md b/docs/Examples/Openai/json_scraper_multi_openai.md deleted file mode 100644 index da31c23c..00000000 --- a/docs/Examples/Openai/json_scraper_multi_openai.md +++ /dev/null @@ -1,38 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - } -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Openai/json_scraper_openai.md b/docs/Examples/Openai/json_scraper_openai.md deleted file mode 100644 index e70316d1..00000000 --- a/docs/Examples/Openai/json_scraper_openai.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/md_scraper_openai.md b/docs/Examples/Openai/md_scraper_openai.md deleted file mode 100644 index 9599feb2..00000000 --- a/docs/Examples/Openai/md_scraper_openai.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using DocumentScraperGraph from MD documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the MD file -# ************************************************ - -FILE_NAME = "inputs/markdown_example.md" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, -} - -# ************************************************ -# Create the DocumentScraperGraph instance and run it -# ************************************************ - -md_scraper_graph = DocumentScraperGraph( - prompt="List me all the projects", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = md_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = md_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/omni_scraper_openai.md b/docs/Examples/Openai/omni_scraper_openai.md deleted file mode 100644 index e60ab057..00000000 --- a/docs/Examples/Openai/omni_scraper_openai.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using OmniScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import OmniScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": True, - "max_images": 5 -} - -# ************************************************ -# Create the OmniScraperGraph instance and run it -# ************************************************ - -omni_scraper_graph = OmniScraperGraph( - prompt="List me all the projects with their titles and image links and descriptions.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = omni_scraper_graph.run() -print(json.dumps(result, indent=2)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = omni_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/omni_search_openai.md b/docs/Examples/Openai/omni_search_openai.md deleted file mode 100644 index bbca43fb..00000000 --- a/docs/Examples/Openai/omni_search_openai.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Example of OmniSearchGraph -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import OmniSearchGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "max_results": 2, - "max_images": 1, - "verbose": True, -} - -# ************************************************ -# Create the OmniSearchGraph instance and run it -# ************************************************ - -omni_search_graph = OmniSearchGraph( - prompt="List me all Chioggia's famous dishes and describe their pictures.", - config=graph_config -) - -result = omni_search_graph.run() -print(json.dumps(result, indent=2)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = omni_search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/rate_limit_openai.md b/docs/Examples/Openai/rate_limit_openai.md deleted file mode 100644 index 5b8345b4..00000000 --- a/docs/Examples/Openai/rate_limit_openai.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - "rate_limit": { - "requests_per_second": 1 - } - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/readme.md b/docs/Examples/Openai/readme.md deleted file mode 100644 index 6c91d8b1..00000000 --- a/docs/Examples/Openai/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -```python -This folder contains an example of how to use ScrapeGraph-AI with OpenAI, an artificial intelligence platform. The examples show how to extract information from a website using a natural language prompt. \ No newline at end of file diff --git a/docs/Examples/Openai/scrape_plain_text_openai.md b/docs/Examples/Openai/scrape_plain_text_openai.md deleted file mode 100644 index 20750b80..00000000 --- a/docs/Examples/Openai/scrape_plain_text_openai.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/screenshot_scraper.md b/docs/Examples/Openai/screenshot_scraper.md deleted file mode 100644 index 8d82ed71..00000000 --- a/docs/Examples/Openai/screenshot_scraper.md +++ /dev/null @@ -1,38 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import ScreenshotScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the ScreenshotScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = ScreenshotScraperGraph( - prompt="List me all the projects", - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Openai/script_generator_openai.md b/docs/Examples/Openai/script_generator_openai.md deleted file mode 100644 index b477bd4c..00000000 --- a/docs/Examples/Openai/script_generator_openai.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "library": "beautifulsoup", - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = ScriptCreatorGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/script_generator_schema_openai.md b/docs/Examples/Openai/script_generator_schema_openai.md deleted file mode 100644 index a9544c99..00000000 --- a/docs/Examples/Openai/script_generator_schema_openai.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o" - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config, - schema=Projects -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/Openai/script_multi_generator_openai.md b/docs/Examples/Openai/script_multi_generator_openai.md deleted file mode 100644 index bf721af4..00000000 --- a/docs/Examples/Openai/script_multi_generator_openai.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "library": "beautifulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/search_graph_openai.md b/docs/Examples/Openai/search_graph_openai.md deleted file mode 100644 index 15296f60..00000000 --- a/docs/Examples/Openai/search_graph_openai.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/Openai/search_graph_schema_openai.md b/docs/Examples/Openai/search_graph_schema_openai.md deleted file mode 100644 index 25349806..00000000 --- a/docs/Examples/Openai/search_graph_schema_openai.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o" - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/search_link_graph_openai.md b/docs/Examples/Openai/search_link_graph_openai.md deleted file mode 100644 index 2f0e6d2d..00000000 --- a/docs/Examples/Openai/search_link_graph_openai.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchLinkGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SearchLinkGraph instance and run it -# ************************************************ - -smart_scraper_graph = SearchLinkGraph( - source="https://sport.sky.it/nba?gr=www", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/smart_scraper_multi_concat_openai.md b/docs/Examples/Openai/smart_scraper_multi_concat_openai.md deleted file mode 100644 index d39511ed..00000000 --- a/docs/Examples/Openai/smart_scraper_multi_concat_openai.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Openai/smart_scraper_multi_openai.md b/docs/Examples/Openai/smart_scraper_multi_openai.md deleted file mode 100644 index 87c227b3..00000000 --- a/docs/Examples/Openai/smart_scraper_multi_openai.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/Openai/smart_scraper_openai.md b/docs/Examples/Openai/smart_scraper_openai.md deleted file mode 100644 index ebb44e93..00000000 --- a/docs/Examples/Openai/smart_scraper_openai.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="Extract me all the articles", - source="https://www.wired.com", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/smart_scraper_schema_openai.md b/docs/Examples/Openai/smart_scraper_schema_openai.md deleted file mode 100644 index 60ffbf05..00000000 --- a/docs/Examples/Openai/smart_scraper_schema_openai.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o-mini", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) diff --git a/docs/Examples/Openai/speech_graph_openai.md b/docs/Examples/Openai/speech_graph_openai.md deleted file mode 100644 index b602f250..00000000 --- a/docs/Examples/Openai/speech_graph_openai.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SpeechSummaryGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SpeechGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define audio output path -# ************************************************ - -FILE_NAME = "website_summary.mp3" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -output_path = os.path.join(curr_dir, FILE_NAME) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - "temperature": 0.7, - }, - "tts_model": { - "api_key": openai_key, - "model": "tts-1", - "voice": "alloy" - }, - "output_path": output_path, -} - -# ************************************************ -# Create the SpeechGraph instance and run it -# ************************************************ - -speech_graph = SpeechGraph( - prompt="Make a detailed audio summary of the projects.", - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = speech_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = speech_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/Openai/xml_scraper_graph_multi_openai.md b/docs/Examples/Openai/xml_scraper_graph_multi_openai.md deleted file mode 100644 index f183bd0f..00000000 --- a/docs/Examples/Openai/xml_scraper_graph_multi_openai.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key":openai_key, - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/Openai/xml_scraper_openai.md b/docs/Examples/Openai/xml_scraper_openai.md deleted file mode 100644 index f61385a9..00000000 --- a/docs/Examples/Openai/xml_scraper_openai.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "verbose":False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/anthropic/document_scraper_anthropic.md b/docs/Examples/anthropic/document_scraper_anthropic.md deleted file mode 100644 index 46e954c2..00000000 --- a/docs/Examples/anthropic/document_scraper_anthropic.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - } -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/anthropic/smart_scraper_lite_anthropic.md b/docs/Examples/anthropic/smart_scraper_lite_anthropic.md deleted file mode 100644 index 406e0023..00000000 --- a/docs/Examples/anthropic/smart_scraper_lite_anthropic.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/anthropic/smart_scraper_multi_lite_anthropic.md b/docs/Examples/anthropic/smart_scraper_multi_lite_anthropic.md deleted file mode 100644 index cdaf312b..00000000 --- a/docs/Examples/anthropic/smart_scraper_multi_lite_anthropic.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/azure/document_scraper_azure.md b/docs/Examples/azure/document_scraper_azure.md deleted file mode 100644 index ba4e9dea..00000000 --- a/docs/Examples/azure/document_scraper_azure.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/azure/smart_scraper_lite_azure.md b/docs/Examples/azure/smart_scraper_lite_azure.md deleted file mode 100644 index fa6134ba..00000000 --- a/docs/Examples/azure/smart_scraper_lite_azure.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/azure/smart_scraper_multi_lite_azure.md b/docs/Examples/azure/smart_scraper_multi_lite_azure.md deleted file mode 100644 index 975b8057..00000000 --- a/docs/Examples/azure/smart_scraper_multi_lite_azure.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.environ["AZURE_OPENAI_KEY"], - "model": "azure_openai/gpt-4o" - }, - "verbose": True, - "headless": False -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/bedrock/document_scraper_bedrock.md b/docs/Examples/bedrock/document_scraper_bedrock.md deleted file mode 100644 index 0642c8ba..00000000 --- a/docs/Examples/bedrock/document_scraper_bedrock.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/bedrock/smart_scraper_lite_bedrock.md b/docs/Examples/bedrock/smart_scraper_lite_bedrock.md deleted file mode 100644 index d8a45b6f..00000000 --- a/docs/Examples/bedrock/smart_scraper_lite_bedrock.md +++ /dev/null @@ -1,27 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import json -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/bedrock/smart_scraper_multi_lite_bedrock.md b/docs/Examples/bedrock/smart_scraper_multi_lite_bedrock.md deleted file mode 100644 index 8e16a7e4..00000000 --- a/docs/Examples/bedrock/smart_scraper_multi_lite_bedrock.md +++ /dev/null @@ -1,30 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import json -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -graph_config = { - "llm": { - "client": "client_name", - "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - "temperature": 0.0 - } -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/GenerateScraper/Readme.md b/docs/Examples/benchmarks/GenerateScraper/Readme.md deleted file mode 100644 index f5a5023f..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/Readme.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -# Local models -# Local models -The two websites benchmark are: -- Example 1: https://perinim.github.io/projects -- Example 2: https://www.wired.com (at 17/4/2024) - -Both are strored locally as txt file in .txt format because in this way we do not have to think about the internet connection - -The time is measured in seconds - -The model runned for this benchmark is Mistral on Ollama with nomic-embed-text - -| Hardware | Model | Example 1 | Example 2 | -| ---------------------- | --------------------------------------- | --------- | --------- | -| Macbook 14' m1 pro | Mistral on Ollama with nomic-embed-text | 30.54s | 35.76s | -| Macbook m2 max | Mistral on Ollama with nomic-embed-text | | | -| Macbook 14' m1 pro
| Llama3 on Ollama with nomic-embed-text | 27.82s | 29.986s | -| Macbook m2 max
| Llama3 on Ollama with nomic-embed-text | | | - - -**Note**: the examples on Docker are not runned on other devices than the Macbook because the performance are to slow (10 times slower than Ollama). -# Performance on APIs services -### Example 1: personal portfolio -**URL**: https://perinim.github.io/projects -**Task**: List me all the projects with their description. - -| Name | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD | -| --------------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- | -| gpt-3.5-turbo | 24.21 | 1892 | 1802 | 90 | 1 | 0.002883 | -| gpt-4-turbo-preview | 6.614 | 1936 | 1802 | 134 | 1 | 0.02204 | -| Grooq with nomic-embed-text | 6.71 | 2201 | 2024 | 177 | 1 | 0 | - -### Example 2: Wired -**URL**: https://www.wired.com -**Task**: List me all the articles with their description. - -| Name | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD | -| --------------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- | -| gpt-3.5-turbo | | | | | | | -| gpt-4-turbo-preview | | | | | | | -| Grooq with nomic-embed-text | | | | | | | - - diff --git a/docs/Examples/benchmarks/GenerateScraper/benchmark_docker.md b/docs/Examples/benchmarks/GenerateScraper/benchmark_docker.md deleted file mode 100644 index ae6fe1dd..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/benchmark_docker.md +++ /dev/null @@ -1 +0,0 @@ -```python diff --git a/docs/Examples/benchmarks/GenerateScraper/benchmark_groq.md b/docs/Examples/benchmarks/GenerateScraper/benchmark_groq.md deleted file mode 100644 index 836d0f6d..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/benchmark_groq.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "headless": False, - "library": "beautifoulsoup" -} - - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = ScriptCreatorGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/GenerateScraper/benchmark_llama3.md b/docs/Examples/benchmarks/GenerateScraper/benchmark_llama3.md deleted file mode 100644 index 81ef1ddf..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/benchmark_llama3.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - # "model_tokens": 2000, # set context length arbitrarily, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "library": "beautifoulsoup" -} - - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = ScriptCreatorGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/GenerateScraper/benchmark_mistral.md b/docs/Examples/benchmarks/GenerateScraper/benchmark_mistral.md deleted file mode 100644 index ab6660ac..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/benchmark_mistral.md +++ /dev/null @@ -1,63 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("GPT4_KEY") - - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - # "model_tokens": 2000, # set context length arbitrarily, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "library": "beautifoulsoup" -} - - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = ScriptCreatorGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/GenerateScraper/benchmark_openai_gpt35.md b/docs/Examples/benchmarks/GenerateScraper/benchmark_openai_gpt35.md deleted file mode 100644 index 9e4bdf5c..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/benchmark_openai_gpt35.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-3.5-turbo", - }, - "library": "beautifoulsoup" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = ScriptCreatorGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/GenerateScraper/benchmark_openai_gpt4.md b/docs/Examples/benchmarks/GenerateScraper/benchmark_openai_gpt4.md deleted file mode 100644 index 70668a3e..00000000 --- a/docs/Examples/benchmarks/GenerateScraper/benchmark_openai_gpt4.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4-turbo-2024-04-09", - }, - "library": "beautifoulsoup" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = ScriptCreatorGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/Readme.md b/docs/Examples/benchmarks/SmartScraper/Readme.md deleted file mode 100644 index 2ed6356d..00000000 --- a/docs/Examples/benchmarks/SmartScraper/Readme.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -# Local models -# Local models -The two websites benchmark are: -- Example 1: https://perinim.github.io/projects -- Example 2: https://www.wired.com (at 17/4/2024) - -Both are strored locally as txt file in .txt format because in this way we do not have to think about the internet connection - -| Hardware | Model | Example 1 | Example 2 | -| ---------------------- | --------------------------------------- | --------- | --------- | -| Macbook 14' m1 pro | Mistral on Ollama with nomic-embed-text | 16.291s | 38.74s | -| Macbook m2 max | Mistral on Ollama with nomic-embed-text | | | -| Macbook 14' m1 pro
| Llama3 on Ollama with nomic-embed-text | 12.88s | 13.84s | -| Macbook m2 max
| Llama3 on Ollama with nomic-embed-text | | | - -**Note**: the examples on Docker are not runned on other devices than the Macbook because the performance are to slow (10 times slower than Ollama). Indeed the results are the following: - -| Hardware | Example 1 | Example 2 | -| ------------------ | --------- | --------- | -| Macbook 14' m1 pro | 139.89 | Too long | -# Performance on APIs services -### Example 1: personal portfolio -**URL**: https://perinim.github.io/projects -**Task**: List me all the projects with their description. - -| Name | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD | -| ------------------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- | -| gpt-3.5-turbo | 4.132s | 438 | 303 | 135 | 1 | 0.000724 | -| gpt-4-turbo-preview | 6.965s | 442 | 303 | 139 | 1 | 0.0072 | -| gpt-4-o | 4.446s | 444 | 305 | 139 | 1 | 0 | -| Grooq with nomic-embed-text
| 1.335s | 648 | 482 | 166 | 1 | 0 | - -### Example 2: Wired -**URL**: https://www.wired.com -**Task**: List me all the articles with their description. - -| Name | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD | -| ------------------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- | -| gpt-3.5-turbo | 8.836s | 1167 | 726 | 441 | 1 | 0.001971 | -| gpt-4-turbo-preview | 21.53s | 1205 | 726 | 479 | 1 | 0.02163 | -| gpt-4-o | 15.27s | 1400 | 715 | 685 | 1 | 0 | -| Grooq with nomic-embed-text
| 3.82s | 2459 | 2192 | 267 | 1 | 0 | diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_docker.md b/docs/Examples/benchmarks/SmartScraper/benchmark_docker.md deleted file mode 100644 index 8e062e9a..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_docker.md +++ /dev/null @@ -1,52 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_groq.md b/docs/Examples/benchmarks/SmartScraper/benchmark_groq.md deleted file mode 100644 index 59b2a9bc..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_groq.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_llama3.md b/docs/Examples/benchmarks/SmartScraper/benchmark_llama3.md deleted file mode 100644 index ab44d449..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_llama3.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_mistral.md b/docs/Examples/benchmarks/SmartScraper/benchmark_mistral.md deleted file mode 100644 index 1f3ba6d1..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_mistral.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - } -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt35.md b/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt35.md deleted file mode 100644 index 77e8088a..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt35.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-3.5-turbo", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt4.md b/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt4.md deleted file mode 100644 index c83fc8d4..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt4.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4-turbo", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt4o.md b/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt4o.md deleted file mode 100644 index a2d9eaa8..00000000 --- a/docs/Examples/benchmarks/SmartScraper/benchmark_openai_gpt4o.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ -files = ["inputs/example_1.txt", "inputs/example_2.txt"] -tasks = ["List me all the projects with their description.", - "List me all the articles with their description."] - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -for i in range(0, 2): - with open(files[i], 'r', encoding="utf-8") as file: - text = file.read() - - smart_scraper_graph = SmartScraperGraph( - prompt=tasks[i], - source=text, - config=graph_config - ) - - result = smart_scraper_graph.run() - print(result) - # ************************************************ - # Get graph execution info - # ************************************************ - - graph_exec_info = smart_scraper_graph.get_execution_info() - print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/benchmarks/readme.md b/docs/Examples/benchmarks/readme.md deleted file mode 100644 index 5d6f24c3..00000000 --- a/docs/Examples/benchmarks/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -```python -These 2 subfolders contain all the scripts and performance documents for the 2 graphs used for the scrapers. -In particular: -* __GenerateScraper__: contains the benchmarks for GenerateScraper class -* __SmartScraper__: contains the benchamrks for SmartScraper class \ No newline at end of file diff --git a/docs/Examples/deepseek/document_scraper_deepseek.md b/docs/Examples/deepseek/document_scraper_deepseek.md deleted file mode 100644 index 45b32ec1..00000000 --- a/docs/Examples/deepseek/document_scraper_deepseek.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek/deepseek-chat", - "api_key": deepseek_key, - }, - "verbose": True, -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/deepseek/smart_scraper_lite_deepseek.md b/docs/Examples/deepseek/smart_scraper_lite_deepseek.md deleted file mode 100644 index 48331ffb..00000000 --- a/docs/Examples/deepseek/smart_scraper_lite_deepseek.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("DEEPSEEK_API_KEY"), - "model": "deepseek/deepseek-coder-33b-instruct", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/deepseek/smart_scraper_multi_lite_deepseek.md b/docs/Examples/deepseek/smart_scraper_multi_lite_deepseek.md deleted file mode 100644 index bc36c2a7..00000000 --- a/docs/Examples/deepseek/smart_scraper_multi_lite_deepseek.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("DEEPSEEK_API_KEY"), - "model": "deepseek/deepseek-coder-33b-instruct", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/ernie/document_scraper_anthropic_ernie.md b/docs/Examples/ernie/document_scraper_anthropic_ernie.md deleted file mode 100644 index 41e40d51..00000000 --- a/docs/Examples/ernie/document_scraper_anthropic_ernie.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from scrapegraphai.graphs import DocumentScraperGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "model": "ernie/ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - } -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/ernie/smart_scraper_lite_ernie.md b/docs/Examples/ernie/smart_scraper_lite_ernie.md deleted file mode 100644 index 73f6530e..00000000 --- a/docs/Examples/ernie/smart_scraper_lite_ernie.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ERNIE_API_KEY"), - "model": "ernie/ernie-bot-4", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/ernie/smart_scraper_multi_lite_ernie.md b/docs/Examples/ernie/smart_scraper_multi_lite_ernie.md deleted file mode 100644 index 8427e7d7..00000000 --- a/docs/Examples/ernie/smart_scraper_multi_lite_ernie.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ERNIE_API_KEY"), - "model": "ernie/ernie-bot-4", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/extras/browser_base_integration.md b/docs/Examples/extras/browser_base_integration.md deleted file mode 100644 index 276c8d12..00000000 --- a/docs/Examples/extras/browser_base_integration.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "browser_base": { - "api_key": os.getenv("BROWSER_BASE_API_KEY"), - "project_id": os.getenv("BROWSER_BASE_PROJECT_ID"), - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/cond_smartscraper_usage.md b/docs/Examples/extras/cond_smartscraper_usage.md deleted file mode 100644 index cf6700ec..00000000 --- a/docs/Examples/extras/cond_smartscraper_usage.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraperMultiConcatGraph with Groq -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("GROQ_APIKEY"), - "model": "groq/gemma-7b-it", - }, - "verbose": True, - "headless": True, - "reattempt": True #Setting this to True will allow the graph to reattempt the scraping process -} - -# ******************************************************* -# Create the SmartScraperMultiCondGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/extras/conditional_usage.md b/docs/Examples/extras/conditional_usage.md deleted file mode 100644 index 2d27e388..00000000 --- a/docs/Examples/extras/conditional_usage.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraperMultiConcatGraph with Groq -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiCondGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source=[ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/extras/custom_prompt.md b/docs/Examples/extras/custom_prompt.md deleted file mode 100644 index df5b54a8..00000000 --- a/docs/Examples/extras/custom_prompt.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -prompt = "Some more info" - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-3.5-turbo", - }, - "additional_info": prompt, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/force_mode.md b/docs/Examples/extras/force_mode.md deleted file mode 100644 index fb5af97c..00000000 --- a/docs/Examples/extras/force_mode.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - # "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "force": True, - "caching": True -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/html_mode.md b/docs/Examples/extras/html_mode.md deleted file mode 100644 index c9e776dc..00000000 --- a/docs/Examples/extras/html_mode.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -By default smart scraper converts in md format the -code. If you want to just use the original code, you have -to specify in the confi -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "html_mode": True, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/load_yml.md b/docs/Examples/extras/load_yml.md deleted file mode 100644 index 6c27333d..00000000 --- a/docs/Examples/extras/load_yml.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import yaml -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -with open("example.yml", 'r') as file: - graph_config = yaml.safe_load(file) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the titles", - source="https://sport.sky.it/nba?gr=www", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/no_cut.md b/docs/Examples/extras/no_cut.md deleted file mode 100644 index 0937f9bf..00000000 --- a/docs/Examples/extras/no_cut.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -This example shows how to do not process the html code in the fetch phase -""" - -import os, json -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": "s", - "model": "openai/gpt-3.5-turbo", - }, - "cut": False, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="Extract me the python code inside the page", - source="https://www.exploit-db.com/exploits/51447", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/proxy_rotation.md b/docs/Examples/extras/proxy_rotation.md deleted file mode 100644 index b26f63e7..00000000 --- a/docs/Examples/extras/proxy_rotation.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "API_KEY", - "model": "openai/gpt-3.5-turbo", - }, - "loader_kwargs": { - "proxy" : { - "server": "http:/**********", - "username": "********", - "password": "***", - }, - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/rag_caching.md b/docs/Examples/extras/rag_caching.md deleted file mode 100644 index d4825cf6..00000000 --- a/docs/Examples/extras/rag_caching.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-3.5-turbo", - }, - "caching": True -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) \ No newline at end of file diff --git a/docs/Examples/extras/reasoning.md b/docs/Examples/extras/reasoning.md deleted file mode 100644 index 11545a74..00000000 --- a/docs/Examples/extras/reasoning.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "reasoning": True, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/extras/scrape_do.md b/docs/Examples/extras/scrape_do.md deleted file mode 100644 index 0aa49357..00000000 --- a/docs/Examples/extras/scrape_do.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "scrape_do": { - "api_key": os.getenv("SCRAPE_DO_API_KEY"), - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects", - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/extras/screenshot_scaping.md b/docs/Examples/extras/screenshot_scaping.md deleted file mode 100644 index 14965162..00000000 --- a/docs/Examples/extras/screenshot_scaping.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -example of scraping with screenshots -""" -import asyncio -from scrapegraphai.utils.screenshot_scraping import (take_screenshot, - select_area_with_opencv, - crop_image, detect_text) - -# STEP 1: Take a screenshot -image = asyncio.run(take_screenshot( - url="https://colab.google/", - save_path="Savedscreenshots/test_image.jpeg", - quality = 50 -)) - -# STEP 2 (Optional): Select an area of the image which you want to use for text detection. -LEFT, TOP, RIGHT, BOTTOM = select_area_with_opencv(image) -print("LEFT: ", LEFT, " TOP: ", TOP, " RIGHT: ", RIGHT, " BOTTOM: ", BOTTOM) - -# STEP 3 (Optional): Crop the image. -# Note: If any of the coordinates (LEFT, TOP, RIGHT, BOTTOM) is None, -# it will be set to the corresponding edge of the image. -cropped_image = crop_image(image, LEFT=LEFT, RIGHT=RIGHT,TOP=TOP,BOTTOM=BOTTOM) - -# STEP 4: Detect text -TEXT = detect_text( - cropped_image, # The image to detect text from - languages = ["en"] # The languages to detect text in -) - -print("DETECTED TEXT: ") -print(TEXT) diff --git a/docs/Examples/extras/serch_graph_scehma.md b/docs/Examples/extras/serch_graph_scehma.md deleted file mode 100644 index 4fab5b3a..00000000 --- a/docs/Examples/extras/serch_graph_scehma.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from pydantic import BaseModel, Field -from typing import List -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -class CeoName(BaseModel): - ceo_name: str = Field(description="The name and surname of the ceo") - -class Ceos(BaseModel): - names: List[CeoName] - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt=f"Who is the ceo of Appke?", - schema = Ceos, - config=graph_config, -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/extras/slow_mo.md b/docs/Examples/extras/slow_mo.md deleted file mode 100644 index f239fe6c..00000000 --- a/docs/Examples/extras/slow_mo.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "loader_kwargs": { - "slow_mo": 10000 - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the titles", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) \ No newline at end of file diff --git a/docs/Examples/extras/undected_playwrigth.md b/docs/Examples/extras/undected_playwrigth.md deleted file mode 100644 index 365d764f..00000000 --- a/docs/Examples/extras/undected_playwrigth.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "headless": False, - "backend": "undetected_chromedriver" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/code_generator_graph_fireworks.md b/docs/Examples/fireworks/code_generator_graph_fireworks.md deleted file mode 100644 index d014e0ac..00000000 --- a/docs/Examples/fireworks/code_generator_graph_fireworks.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -import json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/fireworks/csv_scraper_fireworks.md b/docs/Examples/fireworks/csv_scraper_fireworks.md deleted file mode 100644 index 74e0ccec..00000000 --- a/docs/Examples/fireworks/csv_scraper_fireworks.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "headless": False, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/fireworks/csv_scraper_graph_multi_fireworks.md b/docs/Examples/fireworks/csv_scraper_graph_multi_fireworks.md deleted file mode 100644 index 2b939890..00000000 --- a/docs/Examples/fireworks/csv_scraper_graph_multi_fireworks.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/fireworks/custom_graph_fireworks.md b/docs/Examples/fireworks/custom_graph_fireworks.md deleted file mode 100644 index 40279dad..00000000 --- a/docs/Examples/fireworks/custom_graph_fireworks.md +++ /dev/null @@ -1,99 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" -import os -from dotenv import load_dotenv -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) - -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/fireworks/depth_search_graph_fireworks.md b/docs/Examples/fireworks/depth_search_graph_fireworks.md deleted file mode 100644 index b1a69808..00000000 --- a/docs/Examples/fireworks/depth_search_graph_fireworks.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/fireworks/document_scraper_anthropic_fireworks.md b/docs/Examples/fireworks/document_scraper_anthropic_fireworks.md deleted file mode 100644 index 9b1ecc15..00000000 --- a/docs/Examples/fireworks/document_scraper_anthropic_fireworks.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/fireworks/json_scraper_fireworkspy.md b/docs/Examples/fireworks/json_scraper_fireworkspy.md deleted file mode 100644 index e57ea407..00000000 --- a/docs/Examples/fireworks/json_scraper_fireworkspy.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/fireworks/json_scraper_multi_fireworks.md b/docs/Examples/fireworks/json_scraper_multi_fireworks.md deleted file mode 100644 index e9f4128f..00000000 --- a/docs/Examples/fireworks/json_scraper_multi_fireworks.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/fireworks/rate_limit_fireworks.md b/docs/Examples/fireworks/rate_limit_fireworks.md deleted file mode 100644 index 426816af..00000000 --- a/docs/Examples/fireworks/rate_limit_fireworks.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct", - "rate_limit": { - "requests_per_second": 1 - }, - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/scrape_plain_text_fireworks.md b/docs/Examples/fireworks/scrape_plain_text_fireworks.md deleted file mode 100644 index 31f866d2..00000000 --- a/docs/Examples/fireworks/scrape_plain_text_fireworks.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/script_generator_fireworks.md b/docs/Examples/fireworks/script_generator_fireworks.md deleted file mode 100644 index 5d6aabb8..00000000 --- a/docs/Examples/fireworks/script_generator_fireworks.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, - "library": "beautifulsoup" - -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/script_generator_schema_fireworks.md b/docs/Examples/fireworks/script_generator_schema_fireworks.md deleted file mode 100644 index 9ce34ebe..00000000 --- a/docs/Examples/fireworks/script_generator_schema_fireworks.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "library": "beautifulsoup", -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config, - schema=Projects -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/fireworks/script_multi_generator_fireworks.md b/docs/Examples/fireworks/script_multi_generator_fireworks.md deleted file mode 100644 index 59224069..00000000 --- a/docs/Examples/fireworks/script_multi_generator_fireworks.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "library": "beautifulsoup", -} -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/search_graph_fireworks.md b/docs/Examples/fireworks/search_graph_fireworks.md deleted file mode 100644 index b066769c..00000000 --- a/docs/Examples/fireworks/search_graph_fireworks.md +++ /dev/null @@ -1,37 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "max_results": 2, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/fireworks/search_graph_schema_fireworks.md b/docs/Examples/fireworks/search_graph_schema_fireworks.md deleted file mode 100644 index 3e2c544a..00000000 --- a/docs/Examples/fireworks/search_graph_schema_fireworks.md +++ /dev/null @@ -1,63 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "max_results": 2, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/fireworks/search_link_graph_fireworks.md b/docs/Examples/fireworks/search_link_graph_fireworks.md deleted file mode 100644 index 87ee6fcc..00000000 --- a/docs/Examples/fireworks/search_link_graph_fireworks.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "max_results": 2, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/fireworks/smart_scraper_fireworks.md b/docs/Examples/fireworks/smart_scraper_fireworks.md deleted file mode 100644 index 56ac9747..00000000 --- a/docs/Examples/fireworks/smart_scraper_fireworks.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config, -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/smart_scraper_lite_fireworks.md b/docs/Examples/fireworks/smart_scraper_lite_fireworks.md deleted file mode 100644 index 3c175e6d..00000000 --- a/docs/Examples/fireworks/smart_scraper_lite_fireworks.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("FIREWORKS_API_KEY"), - "model": "fireworks/llama-v2-70b-chat", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/fireworks/smart_scraper_multi_concat_fireworks.md b/docs/Examples/fireworks/smart_scraper_multi_concat_fireworks.md deleted file mode 100644 index 3a933a70..00000000 --- a/docs/Examples/fireworks/smart_scraper_multi_concat_fireworks.md +++ /dev/null @@ -1,38 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/fireworks/smart_scraper_multi_fireworks.md b/docs/Examples/fireworks/smart_scraper_multi_fireworks.md deleted file mode 100644 index e239273f..00000000 --- a/docs/Examples/fireworks/smart_scraper_multi_fireworks.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/fireworks/smart_scraper_multi_lite_fireworks.md b/docs/Examples/fireworks/smart_scraper_multi_lite_fireworks.md deleted file mode 100644 index e17d9e66..00000000 --- a/docs/Examples/fireworks/smart_scraper_multi_lite_fireworks.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("FIREWORKS_API_KEY"), - "model": "fireworks/llama-v2-70b-chat", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/fireworks/smart_scraper_schema_fireworks.md b/docs/Examples/fireworks/smart_scraper_schema_fireworks.md deleted file mode 100644 index b020816e..00000000 --- a/docs/Examples/fireworks/smart_scraper_schema_fireworks.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) diff --git a/docs/Examples/fireworks/xml_scraper_fireworks.md b/docs/Examples/fireworks/xml_scraper_fireworks.md deleted file mode 100644 index 5c9fee07..00000000 --- a/docs/Examples/fireworks/xml_scraper_fireworks.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/fireworks/xml_scraper_graph_multi_fireworks.md b/docs/Examples/fireworks/xml_scraper_graph_multi_fireworks.md deleted file mode 100644 index 4747050c..00000000 --- a/docs/Examples/fireworks/xml_scraper_graph_multi_fireworks.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -fireworks_api_key = os.getenv("FIREWORKS_APIKEY") - -graph_config = { - "llm": { - "api_key": fireworks_api_key, - "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "verbose": True, - "headless": False, -} -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_genai/code_generator_graph_gemini.md b/docs/Examples/google_genai/code_generator_graph_gemini.md deleted file mode 100644 index a5c09b66..00000000 --- a/docs/Examples/google_genai/code_generator_graph_gemini.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/google_genai/csv_scraper_gemini.md b/docs/Examples/google_genai/csv_scraper_gemini.md deleted file mode 100644 index 8024e3c4..00000000 --- a/docs/Examples/google_genai/csv_scraper_gemini.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the csv file -# ************************************************ - -text = pd.read_csv("inputs/username.csv") - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_genai/csv_scraper_graph_multi_gemini.md b/docs/Examples/google_genai/csv_scraper_graph_multi_gemini.md deleted file mode 100644 index 0ff0258a..00000000 --- a/docs/Examples/google_genai/csv_scraper_graph_multi_gemini.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_genai/depth_search_graph_gemini.md b/docs/Examples/google_genai/depth_search_graph_gemini.md deleted file mode 100644 index 633a293d..00000000 --- a/docs/Examples/google_genai/depth_search_graph_gemini.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/google_genai/document_scraper_gemini.md b/docs/Examples/google_genai/document_scraper_gemini.md deleted file mode 100644 index 1baeeba4..00000000 --- a/docs/Examples/google_genai/document_scraper_gemini.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/google_genai/json_scraper_gemini.md b/docs/Examples/google_genai/json_scraper_gemini.md deleted file mode 100644 index 36d5bf42..00000000 --- a/docs/Examples/google_genai/json_scraper_gemini.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/google_genai/json_scraper_multi_gemini.md b/docs/Examples/google_genai/json_scraper_multi_gemini.md deleted file mode 100644 index f60f641e..00000000 --- a/docs/Examples/google_genai/json_scraper_multi_gemini.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, - "library": "beautifulsoup" -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/google_genai/rate_limit_gemini.md b/docs/Examples/google_genai/rate_limit_gemini.md deleted file mode 100644 index a41d5e1b..00000000 --- a/docs/Examples/google_genai/rate_limit_gemini.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -from dotenv import load_dotenv -from scrapegraphai.utils import prettify_exec_info -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - "rate_limit": { - "requests_per_second": 1 - } - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/readme.md b/docs/Examples/google_genai/readme.md deleted file mode 100644 index b1aa385a..00000000 --- a/docs/Examples/google_genai/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -```python -This folder contains an example of how to use ScrapeGraph-AI with Gemini, a large language model (LLM) from Google AI. The example shows how to extract information from a website using a natural language prompt. \ No newline at end of file diff --git a/docs/Examples/google_genai/scrape_plain_text_gemini.md b/docs/Examples/google_genai/scrape_plain_text_gemini.md deleted file mode 100644 index f6a462bb..00000000 --- a/docs/Examples/google_genai/scrape_plain_text_gemini.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - "temperature": 0, - "streaming": True - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/scrape_xml_gemini.md b/docs/Examples/google_genai/scrape_xml_gemini.md deleted file mode 100644 index 003be1e1..00000000 --- a/docs/Examples/google_genai/scrape_xml_gemini.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - "temperature": 0, - "streaming": True - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/script_generator_gemini.md b/docs/Examples/google_genai/script_generator_gemini.md deleted file mode 100644 index 276ac95f..00000000 --- a/docs/Examples/google_genai/script_generator_gemini.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, - "library": "beautifoulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -smart_scraper_graph = ScriptCreatorGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/script_multi_generator_gemini.md b/docs/Examples/google_genai/script_multi_generator_gemini.md deleted file mode 100644 index 670be810..00000000 --- a/docs/Examples/google_genai/script_multi_generator_gemini.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, - "library": "beautifoulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/search_graph_gemini.md b/docs/Examples/google_genai/search_graph_gemini.md deleted file mode 100644 index b5792a83..00000000 --- a/docs/Examples/google_genai/search_graph_gemini.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - "temperature": 0, - "streaming": True - }, - "max_results": 5, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me all the regions of Italy.", - config=graph_config -) - -result = search_graph.run() -print(result) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_genai/search_graph_schema_gemini.md b/docs/Examples/google_genai/search_graph_schema_gemini.md deleted file mode 100644 index f4d3d59d..00000000 --- a/docs/Examples/google_genai/search_graph_schema_gemini.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_genai/search_link_graph_gemini.md b/docs/Examples/google_genai/search_link_graph_gemini.md deleted file mode 100644 index 573c3e3e..00000000 --- a/docs/Examples/google_genai/search_link_graph_gemini.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_genai/smart_scraper_gemini.md b/docs/Examples/google_genai/smart_scraper_gemini.md deleted file mode 100644 index 01493965..00000000 --- a/docs/Examples/google_genai/smart_scraper_gemini.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.utils import prettify_exec_info -from scrapegraphai.graphs import SmartScraperGraph -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/smart_scraper_lite_google_genai.md b/docs/Examples/google_genai/smart_scraper_lite_google_genai.md deleted file mode 100644 index a5c96f0a..00000000 --- a/docs/Examples/google_genai/smart_scraper_lite_google_genai.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("GOOGLE_API_KEY"), - "model": "gemini-pro", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/smart_scraper_multi_concat_gemini.md b/docs/Examples/google_genai/smart_scraper_multi_concat_gemini.md deleted file mode 100644 index 5d977e1c..00000000 --- a/docs/Examples/google_genai/smart_scraper_multi_concat_gemini.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/google_genai/smart_scraper_multi_gemini.md b/docs/Examples/google_genai/smart_scraper_multi_gemini.md deleted file mode 100644 index 495d89cb..00000000 --- a/docs/Examples/google_genai/smart_scraper_multi_gemini.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/google_genai/smart_scraper_multi_lite_gemini.md b/docs/Examples/google_genai/smart_scraper_multi_lite_gemini.md deleted file mode 100644 index ae6fe1dd..00000000 --- a/docs/Examples/google_genai/smart_scraper_multi_lite_gemini.md +++ /dev/null @@ -1 +0,0 @@ -```python diff --git a/docs/Examples/google_genai/smart_scraper_multi_lite_google_genai.md b/docs/Examples/google_genai/smart_scraper_multi_lite_google_genai.md deleted file mode 100644 index c33d0f58..00000000 --- a/docs/Examples/google_genai/smart_scraper_multi_lite_google_genai.md +++ /dev/null @@ -1,35 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("GOOGLE_API_KEY"), - "model": "gemini-pro", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/smart_scraper_schema_gemini.md b/docs/Examples/google_genai/smart_scraper_schema_gemini.md deleted file mode 100644 index 10e992e7..00000000 --- a/docs/Examples/google_genai/smart_scraper_schema_gemini.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.utils import prettify_exec_info -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_genai/xml_scraper_gemini.md b/docs/Examples/google_genai/xml_scraper_gemini.md deleted file mode 100644 index 008d14aa..00000000 --- a/docs/Examples/google_genai/xml_scraper_gemini.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/docs/Examples/google_genai/xml_scraper_graph_multi_gemini.md b/docs/Examples/google_genai/xml_scraper_graph_multi_gemini.md deleted file mode 100644 index 2159cfb7..00000000 --- a/docs/Examples/google_genai/xml_scraper_graph_multi_gemini.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_genai/gemini-pro", - }, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_vertexai/code_generator_graph_vertex.md b/docs/Examples/google_vertexai/code_generator_graph_vertex.md deleted file mode 100644 index 4e1a0e2e..00000000 --- a/docs/Examples/google_vertexai/code_generator_graph_vertex.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" - -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/google_vertexai/csv_scraper_gemini.md b/docs/Examples/google_vertexai/csv_scraper_gemini.md deleted file mode 100644 index a1d75e4d..00000000 --- a/docs/Examples/google_vertexai/csv_scraper_gemini.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" - -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the csv file -# ************************************************ - -text = pd.read_csv("inputs/username.csv") - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_vertexai/csv_scraper_graph_multi_gemini.md b/docs/Examples/google_vertexai/csv_scraper_graph_multi_gemini.md deleted file mode 100644 index a7381d73..00000000 --- a/docs/Examples/google_vertexai/csv_scraper_graph_multi_gemini.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" - -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_vertexai/custom_graph_gemini.md b/docs/Examples/google_vertexai/custom_graph_gemini.md deleted file mode 100644 index c512a7e2..00000000 --- a/docs/Examples/google_vertexai/custom_graph_gemini.md +++ /dev/null @@ -1,85 +0,0 @@ -```python -""" -Example of custom graph using Gemini Google model -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.models import Gemini -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - "temperature": 0, - "streaming": True - }, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = Gemini(graph_config["llm"]) - -# define the nodes for the graph -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={"chunk_size": 4096} -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={"llm": llm_model}, -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={"llm": llm_model}, -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes={ - fetch_node, - parse_node, - rag_node, - generate_answer_node, - }, - edges={ - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - }, - entry_point=fetch_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "List me the projects with their description", - "url": "https://perinim.github.io/projects/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/google_vertexai/depth_search_graph_gemini.md b/docs/Examples/google_vertexai/depth_search_graph_gemini.md deleted file mode 100644 index c4cb60d9..00000000 --- a/docs/Examples/google_vertexai/depth_search_graph_gemini.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/google_vertexai/document_scraper_vertex.md b/docs/Examples/google_vertexai/document_scraper_vertex.md deleted file mode 100644 index 0f0bf3bb..00000000 --- a/docs/Examples/google_vertexai/document_scraper_vertex.md +++ /dev/null @@ -1,42 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/google_vertexai/json_scraper_gemini.md b/docs/Examples/google_vertexai/json_scraper_gemini.md deleted file mode 100644 index 751614e4..00000000 --- a/docs/Examples/google_vertexai/json_scraper_gemini.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/google_vertexai/json_scraper_multi_gemini.md b/docs/Examples/google_vertexai/json_scraper_multi_gemini.md deleted file mode 100644 index e1cffc86..00000000 --- a/docs/Examples/google_vertexai/json_scraper_multi_gemini.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, - "library": "beautifulsoup" -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/google_vertexai/rate_limit_gemini.md b/docs/Examples/google_vertexai/rate_limit_gemini.md deleted file mode 100644 index e7255aa1..00000000 --- a/docs/Examples/google_vertexai/rate_limit_gemini.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.utils import prettify_exec_info -from scrapegraphai.graphs import SmartScraperGraph -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - "rate_limit": { - "requests_per_second": 1 - } - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/readme.md b/docs/Examples/google_vertexai/readme.md deleted file mode 100644 index b1aa385a..00000000 --- a/docs/Examples/google_vertexai/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -```python -This folder contains an example of how to use ScrapeGraph-AI with Gemini, a large language model (LLM) from Google AI. The example shows how to extract information from a website using a natural language prompt. \ No newline at end of file diff --git a/docs/Examples/google_vertexai/scrape_plain_text_gemini.md b/docs/Examples/google_vertexai/scrape_plain_text_gemini.md deleted file mode 100644 index 5a94f748..00000000 --- a/docs/Examples/google_vertexai/scrape_plain_text_gemini.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - "temperature": 0, - "streaming": True - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/scrape_xml_gemini.md b/docs/Examples/google_vertexai/scrape_xml_gemini.md deleted file mode 100644 index 2d55e0de..00000000 --- a/docs/Examples/google_vertexai/scrape_xml_gemini.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - "temperature": 0, - "streaming": True - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/script_generator_gemini.md b/docs/Examples/google_vertexai/script_generator_gemini.md deleted file mode 100644 index b5580613..00000000 --- a/docs/Examples/google_vertexai/script_generator_gemini.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, - "library": "beautifoulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -smart_scraper_graph = ScriptCreatorGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/script_multi_generator_gemini.md b/docs/Examples/google_vertexai/script_multi_generator_gemini.md deleted file mode 100644 index b736fd78..00000000 --- a/docs/Examples/google_vertexai/script_multi_generator_gemini.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, - "library": "beautifoulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/search_graph_gemini.md b/docs/Examples/google_vertexai/search_graph_gemini.md deleted file mode 100644 index cc110619..00000000 --- a/docs/Examples/google_vertexai/search_graph_gemini.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - "temperature": 0, - "streaming": True - }, - "max_results": 5, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me all the regions of Italy.", - config=graph_config -) - -result = search_graph.run() -print(result) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_vertexai/search_graph_schema_gemini.md b/docs/Examples/google_vertexai/search_graph_schema_gemini.md deleted file mode 100644 index 868a4f03..00000000 --- a/docs/Examples/google_vertexai/search_graph_schema_gemini.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from dotenv import load_dotenv -load_dotenv() - -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -from pydantic import BaseModel, Field -from typing import List - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_vertexai/search_link_graph_gemini.md b/docs/Examples/google_vertexai/search_link_graph_gemini.md deleted file mode 100644 index 2fe787fa..00000000 --- a/docs/Examples/google_vertexai/search_link_graph_gemini.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/google_vertexai/smart_scraper_gemini.md b/docs/Examples/google_vertexai/smart_scraper_gemini.md deleted file mode 100644 index 3b7bd42a..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_gemini.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.utils import prettify_exec_info -from scrapegraphai.graphs import SmartScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/smart_scraper_lite_google_vertexai.md b/docs/Examples/google_vertexai/smart_scraper_lite_google_vertexai.md deleted file mode 100644 index 3d8a8fd8..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_lite_google_vertexai.md +++ /dev/null @@ -1,34 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "project": os.getenv("GOOGLE_CLOUD_PROJECT"), - "location": "us-central1", - "model": "text-bison@001", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/google_vertexai/smart_scraper_multi_concat_gemini.md b/docs/Examples/google_vertexai/smart_scraper_multi_concat_gemini.md deleted file mode 100644 index 283095c6..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_multi_concat_gemini.md +++ /dev/null @@ -1,37 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/google_vertexai/smart_scraper_multi_gemini.md b/docs/Examples/google_vertexai/smart_scraper_multi_gemini.md deleted file mode 100644 index 84cf8693..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_multi_gemini.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os, json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/google_vertexai/smart_scraper_multi_lite_google_vertexai.md b/docs/Examples/google_vertexai/smart_scraper_multi_lite_google_vertexai.md deleted file mode 100644 index 4639820f..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_multi_lite_google_vertexai.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "project": os.getenv("GOOGLE_CLOUD_PROJECT"), - "location": "us-central1", - "model": "text-bison@001", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/smart_scraper_multi_lite_vertex.md b/docs/Examples/google_vertexai/smart_scraper_multi_lite_vertex.md deleted file mode 100644 index abbc43fc..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_multi_lite_vertex.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "project": os.getenv("GOOGLE_CLOUD_PROJECT"), - "location": "us-central1", - "model": "text-bison@001", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/smart_scraper_schema_gemini.md b/docs/Examples/google_vertexai/smart_scraper_schema_gemini.md deleted file mode 100644 index a781e656..00000000 --- a/docs/Examples/google_vertexai/smart_scraper_schema_gemini.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" - -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.utils import prettify_exec_info -from scrapegraphai.graphs import SmartScraperGraph -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://www.wired.com", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/google_vertexai/xml_scraper_gemini.md b/docs/Examples/google_vertexai/xml_scraper_gemini.md deleted file mode 100644 index 67b0eb0a..00000000 --- a/docs/Examples/google_vertexai/xml_scraper_gemini.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/docs/Examples/google_vertexai/xml_scraper_graph_multi_gemini.md b/docs/Examples/google_vertexai/xml_scraper_graph_multi_gemini.md deleted file mode 100644 index e0fc6069..00000000 --- a/docs/Examples/google_vertexai/xml_scraper_graph_multi_gemini.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -gemini_key = os.getenv("GOOGLE_APIKEY") - -graph_config = { - "llm": { - "api_key": gemini_key, - "model": "google_vertexai/gemini-1.5-pro", - }, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/groq/document_scraper_groq.md b/docs/Examples/groq/document_scraper_groq.md deleted file mode 100644 index 477ed535..00000000 --- a/docs/Examples/groq/document_scraper_groq.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "verbose": True, - "headless": False -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/groq/smart_scraper_lite_groq.md b/docs/Examples/groq/smart_scraper_lite_groq.md deleted file mode 100644 index 55537173..00000000 --- a/docs/Examples/groq/smart_scraper_lite_groq.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("GROQ_API_KEY"), - "model": "mixtral-8x7b-32768", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/groq/smart_scraper_multi_lite_groq.md b/docs/Examples/groq/smart_scraper_multi_lite_groq.md deleted file mode 100644 index d02d9951..00000000 --- a/docs/Examples/groq/smart_scraper_multi_lite_groq.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("GROQ_API_KEY"), - "model": "mixtral-8x7b-32768", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/huggingfacehub/code_generator_graph_huggingfacehub.md b/docs/Examples/huggingfacehub/code_generator_graph_huggingfacehub.md deleted file mode 100644 index 0d460371..00000000 --- a/docs/Examples/huggingfacehub/code_generator_graph_huggingfacehub.md +++ /dev/null @@ -1,72 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" - -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": { - "model_instance": llm_model_instance - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.md b/docs/Examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.md deleted file mode 100644 index b842ce16..00000000 --- a/docs/Examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.md +++ /dev/null @@ -1,69 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" - -import os -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/huggingfacehub/csv_scraper_huggingfacehub.md b/docs/Examples/huggingfacehub/csv_scraper_huggingfacehub.md deleted file mode 100644 index 31d752b8..00000000 --- a/docs/Examples/huggingfacehub/csv_scraper_huggingfacehub.md +++ /dev/null @@ -1,71 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" - -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/huggingfacehub/custom_graph_huggingfacehub.md b/docs/Examples/huggingfacehub/custom_graph_huggingfacehub.md deleted file mode 100644 index 4f309502..00000000 --- a/docs/Examples/huggingfacehub/custom_graph_huggingfacehub.md +++ /dev/null @@ -1,122 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" - -import os -from dotenv import load_dotenv -from langchain_openai import OpenAIEmbeddings -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = OpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - rag_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/huggingfacehub/depth_search_graph_huggingfacehub.md b/docs/Examples/huggingfacehub/depth_search_graph_huggingfacehub.md deleted file mode 100644 index b062046c..00000000 --- a/docs/Examples/huggingfacehub/depth_search_graph_huggingfacehub.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/huggingfacehub/document_scraper_huggingfacehub.md b/docs/Examples/huggingfacehub/document_scraper_huggingfacehub.md deleted file mode 100644 index 82473c2d..00000000 --- a/docs/Examples/huggingfacehub/document_scraper_huggingfacehub.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -# ************************************************ -# Define the configuration for the graph -# ************************************************ -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/huggingfacehub/json_scraper_huggingfacehub.md b/docs/Examples/huggingfacehub/json_scraper_huggingfacehub.md deleted file mode 100644 index 3f3c5ef0..00000000 --- a/docs/Examples/huggingfacehub/json_scraper_huggingfacehub.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) diff --git a/docs/Examples/huggingfacehub/json_scraper_multi_huggingfacehub.md b/docs/Examples/huggingfacehub/json_scraper_multi_huggingfacehub.md deleted file mode 100644 index 790f1ac8..00000000 --- a/docs/Examples/huggingfacehub/json_scraper_multi_huggingfacehub.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/huggingfacehub/scrape_plain_text_huggingfacehub.md b/docs/Examples/huggingfacehub/scrape_plain_text_huggingfacehub.md deleted file mode 100644 index cfbdafd6..00000000 --- a/docs/Examples/huggingfacehub/scrape_plain_text_huggingfacehub.md +++ /dev/null @@ -1,69 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/huggingfacehub/script_generator_huggingfacehub.md b/docs/Examples/huggingfacehub/script_generator_huggingfacehub.md deleted file mode 100644 index ef7030ca..00000000 --- a/docs/Examples/huggingfacehub/script_generator_huggingfacehub.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') -# ************************************************ -# Initialize the model instances -# ************************************************ - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/huggingfacehub/script_multi_generator_huggingfacehub.md b/docs/Examples/huggingfacehub/script_multi_generator_huggingfacehub.md deleted file mode 100644 index 1722ebb0..00000000 --- a/docs/Examples/huggingfacehub/script_multi_generator_huggingfacehub.md +++ /dev/null @@ -1,67 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/huggingfacehub/search_graph_huggingfacehub.md b/docs/Examples/huggingfacehub/search_graph_huggingfacehub.md deleted file mode 100644 index 42792b14..00000000 --- a/docs/Examples/huggingfacehub/search_graph_huggingfacehub.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/huggingfacehub/search_link_graph_huggingfacehub.md b/docs/Examples/huggingfacehub/search_link_graph_huggingfacehub.md deleted file mode 100644 index 09072b29..00000000 --- a/docs/Examples/huggingfacehub/search_link_graph_huggingfacehub.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/huggingfacehub/smart_scraper_huggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_huggingfacehub.md deleted file mode 100644 index 8af44412..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_huggingfacehub.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - - -## required environment variable in .env -#HUGGINGFACEHUB_API_TOKEN -load_dotenv() - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') -# ************************************************ -# Initialize the model instances -# ************************************************ - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, event_end_date, event_end_time, location, event_mode, event_category, third_party_redirect, no_of_days, time_in_hours, hosted_or_attending, refreshments_type, registration_available, registration_link", - # also accepts a string with the already downloaded HTML code - source="https://www.hmhco.com/event", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - - diff --git a/docs/Examples/huggingfacehub/smart_scraper_lite_huggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_lite_huggingfacehub.md deleted file mode 100644 index 25c27272..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_lite_huggingfacehub.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("HUGGINGFACEHUB_API_TOKEN"), - "model": "huggingfacehub/meta-llama/Llama-2-70b-chat-hf", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/huggingfacehub/smart_scraper_multi_concat_huggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_multi_concat_huggingfacehub.md deleted file mode 100644 index 41571d70..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_multi_concat_huggingfacehub.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') -# ************************************************ -# Initialize the model instances -# ************************************************ - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/huggingfacehub/smart_scraper_multi_huggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_multi_huggingfacehub.md deleted file mode 100644 index 71142b4e..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_multi_huggingfacehub.md +++ /dev/null @@ -1,49 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os, json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/huggingfacehub/smart_scraper_multi_lite_huggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_multi_lite_huggingfacehub.md deleted file mode 100644 index 11a395d8..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_multi_lite_huggingfacehub.md +++ /dev/null @@ -1,35 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("HUGGINGFACEHUB_API_TOKEN"), - "model": "huggingfacehub/meta-llama/Llama-2-70b-chat-hf", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/huggingfacehub/smart_scraper_multi_lite_uhggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_multi_lite_uhggingfacehub.md deleted file mode 100644 index ae6fe1dd..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_multi_lite_uhggingfacehub.md +++ /dev/null @@ -1 +0,0 @@ -```python diff --git a/docs/Examples/huggingfacehub/smart_scraper_schema_huggingfacehub.md b/docs/Examples/huggingfacehub/smart_scraper_schema_huggingfacehub.md deleted file mode 100644 index 300d43a6..00000000 --- a/docs/Examples/huggingfacehub/smart_scraper_schema_huggingfacehub.md +++ /dev/null @@ -1,68 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key -""" - -import os -from dotenv import load_dotenv -from typing import Dict - -from pydantic import BaseModel -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str - description: str - -class Projects(BaseModel): - Projects: Dict[str, Project] - -## required environment variable in .env -#HUGGINGFACEHUB_API_TOKEN -load_dotenv() - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') -# ************************************************ -# Initialize the model instances -# ************************************************ - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.md b/docs/Examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.md deleted file mode 100644 index 643c7aca..00000000 --- a/docs/Examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.md +++ /dev/null @@ -1,68 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" - -import os -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/huggingfacehub/xml_scraper_huggingfacehub.md b/docs/Examples/huggingfacehub/xml_scraper_huggingfacehub.md deleted file mode 100644 index fcf59e71..00000000 --- a/docs/Examples/huggingfacehub/xml_scraper_huggingfacehub.md +++ /dev/null @@ -1,69 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from langchain_community.llms import HuggingFaceEndpoint -from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN') - -repo_id = "mistralai/Mistral-7B-Instruct-v0.2" - -llm_model_instance = HuggingFaceEndpoint( - repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN -) - -embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( - api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2" -) - -graph_config = { - "llm": {"model_instance": llm_model_instance}, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") - diff --git a/docs/Examples/integrations/indexify_node_example.md b/docs/Examples/integrations/indexify_node_example.md deleted file mode 100644 index 5f1db60a..00000000 --- a/docs/Examples/integrations/indexify_node_example.md +++ /dev/null @@ -1,73 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" - -import os, json -from typing import List - -from dotenv import load_dotenv -load_dotenv() - -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.integrations import IndexifyNode - - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Image(BaseModel): - url: str = Field(description="The url of the image") - -class Images(BaseModel): - images: List[Image] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key":openai_key, - "model": "openai/gpt-3.5-turbo", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Define the custom nodes for the graph -# ************************************************ - -indexify_node = IndexifyNode( - input="answer & img_urls", - output=["is_indexed"], - node_config={ - "verbose": True - } -) - -# ************************************************ -# Create the SmartScraperGraph instance -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the images with their url", - source="https://giphy.com/", - schema=Images, - config=graph_config -) - -# Add the custom node to the graph -smart_scraper_graph.append_node(indexify_node) - -# ************************************************ -# Run the SmartScraperGraph -# ************************************************ - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=2)) diff --git a/docs/Examples/local_models/code_generator_graph_ollama.md b/docs/Examples/local_models/code_generator_graph_ollama.md deleted file mode 100644 index 4ad5cee3..00000000 --- a/docs/Examples/local_models/code_generator_graph_ollama.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" - -import json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", - "base_url": "http://localhost:11434", - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/local_models/csv_scraper_graph_multi_ollama.md b/docs/Examples/local_models/csv_scraper_graph_multi_ollama.md deleted file mode 100644 index 937c0315..00000000 --- a/docs/Examples/local_models/csv_scraper_graph_multi_ollama.md +++ /dev/null @@ -1,63 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" - -import os -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, - "verbose": True, -} - -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/local_models/csv_scraper_ollama.md b/docs/Examples/local_models/csv_scraper_ollama.md deleted file mode 100644 index c22080cc..00000000 --- a/docs/Examples/local_models/csv_scraper_ollama.md +++ /dev/null @@ -1,63 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" - -import os -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, - "verbose": True, -} - -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/local_models/custom_graph_ollama.md b/docs/Examples/local_models/custom_graph_ollama.md deleted file mode 100644 index e00dca76..00000000 --- a/docs/Examples/local_models/custom_graph_ollama.md +++ /dev/null @@ -1,102 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" - -import os -from langchain_openai import OpenAIEmbeddings -from langchain_openai import ChatOpenAI -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - - "verbose": True, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) - -# define the nodes for the graph -robot_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={ - "llm_model": llm_model, - "force_scraping": True, - "verbose": True, - } -) - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "verbose": True, - "headless": True, - } -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={ - "chunk_size": 4096, - "verbose": True, - } -) - -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes=[ - robot_node, - fetch_node, - parse_node, - generate_answer_node, - ], - edges=[ - (robot_node, fetch_node), - (fetch_node, parse_node), - (parse_node, generate_answer_node) - ], - entry_point=robot_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "Describe the content", - "url": "https://example.com/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/docs/Examples/local_models/depth_search_graph_ollama.md b/docs/Examples/local_models/depth_search_graph_ollama.md deleted file mode 100644 index 32781acf..00000000 --- a/docs/Examples/local_models/depth_search_graph_ollama.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/local_models/document_scraper_ollama.md b/docs/Examples/local_models/document_scraper_ollama.md deleted file mode 100644 index f9eadeeb..00000000 --- a/docs/Examples/local_models/document_scraper_ollama.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -document_scraper example -""" -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - "model_tokens": 4000, - }, - "verbose": True, - "headless": False, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/local_models/json_scraper_multi_ollama.md b/docs/Examples/local_models/json_scraper_multi_ollama.md deleted file mode 100644 index 1a0d100c..00000000 --- a/docs/Examples/local_models/json_scraper_multi_ollama.md +++ /dev/null @@ -1,37 +0,0 @@ -```python -""" -Module for showing how PDFScraper multi works -""" -import os -import json -from scrapegraphai.graphs import JSONScraperMultiGraph - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - "model_tokens": 4000, - }, - "verbose": True, - "headless": False, -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/local_models/json_scraper_ollama.md b/docs/Examples/local_models/json_scraper_ollama.md deleted file mode 100644 index 4795d7a8..00000000 --- a/docs/Examples/local_models/json_scraper_ollama.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "verbose": True, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/local_models/package-lock.md b/docs/Examples/local_models/package-lock.md deleted file mode 100644 index 5c692a57..00000000 --- a/docs/Examples/local_models/package-lock.md +++ /dev/null @@ -1,7 +0,0 @@ -```python -{ - "name": "local_models", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/docs/Examples/local_models/package.md b/docs/Examples/local_models/package.md deleted file mode 100644 index 15fef0d4..00000000 --- a/docs/Examples/local_models/package.md +++ /dev/null @@ -1,2 +0,0 @@ -```python -{} diff --git a/docs/Examples/local_models/scrape_plain_text_ollama.md b/docs/Examples/local_models/scrape_plain_text_ollama.md deleted file mode 100644 index 663dabc4..00000000 --- a/docs/Examples/local_models/scrape_plain_text_ollama.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" - -import os -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/local_models/script_generator_ollama.md b/docs/Examples/local_models/script_generator_ollama.md deleted file mode 100644 index 053d6b60..00000000 --- a/docs/Examples/local_models/script_generator_ollama.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0.5, - # "model_tokens": 2000, # set context length arbitrarily, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "library": "beautifoulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -smart_scraper_graph = ScriptCreatorGraph( - prompt="List me all the news with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/local_models/script_multi_generator_ollama.md b/docs/Examples/local_models/script_multi_generator_ollama.md deleted file mode 100644 index 42ee2134..00000000 --- a/docs/Examples/local_models/script_multi_generator_ollama.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - # "model_tokens": 2000, # set context length arbitrarily, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "library": "beautifoulsoup", - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/local_models/search_graph_ollama.md b/docs/Examples/local_models/search_graph_ollama.md deleted file mode 100644 index 464a7311..00000000 --- a/docs/Examples/local_models/search_graph_ollama.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - # "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "max_results": 5, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/local_models/search_graph_schema_ollama.md b/docs/Examples/local_models/search_graph_schema_ollama.md deleted file mode 100644 index 3912ce61..00000000 --- a/docs/Examples/local_models/search_graph_schema_ollama.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Example of Search Graph -""" -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -from pydantic import BaseModel, Field -from typing import List - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/local_models/search_link_graph_ollama.md b/docs/Examples/local_models/search_link_graph_ollama.md deleted file mode 100644 index fe4d364e..00000000 --- a/docs/Examples/local_models/search_link_graph_ollama.md +++ /dev/null @@ -1,52 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from scrapegraphai.graphs import SearchLinkGraph -from scrapegraphai.utils import prettify_exec_info -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3.1:8b", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - - "verbose": True, - "headless": False, - "filter_config": { - "diff_domain_filter": True, - # "img_exts": ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp', '.ico'], - # "lang_indicators": ['lang=', '/fr', '/pt', '/es', '/de', '/jp', '/it'], - # "irrelevant_keywords": [ - # '/login', '/signup', '/register', '/contact', 'facebook.com', 'twitter.com', - # 'linkedin.com', 'instagram.com', '.js', '.css', '/wp-content/', '/wp-admin/', - # '/wp-includes/', '/wp-json/', '/wp-comments-post.php', ';amp', '/about', - # '/careers', '/jobs', '/privacy', '/terms', '/legal', '/faq', '/help', - # '.pdf', '.zip', '/news', '/files', '/downloads' - # ] - }, -} - -# ************************************************ -# Create the SearchLinkGraph instance and run it -# ************************************************ - -smart_scraper_graph = SearchLinkGraph( - source="https://sport.sky.it/nba?gr=www", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/local_models/smart_scraper_lite_ollama.md b/docs/Examples/local_models/smart_scraper_lite_ollama.md deleted file mode 100644 index 7bfc0654..00000000 --- a/docs/Examples/local_models/smart_scraper_lite_ollama.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper - -""" -import json -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", - "base_url": "http://localhost:11434", - }, - "verbose": True, - "headless": False -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/local_models/smart_scraper_multi_concat_ollama.md b/docs/Examples/local_models/smart_scraper_multi_concat_ollama.md deleted file mode 100644 index 5b05935d..00000000 --- a/docs/Examples/local_models/smart_scraper_multi_concat_ollama.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, - "headless": False -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/local_models/smart_scraper_multi_lite_ollama.md b/docs/Examples/local_models/smart_scraper_multi_lite_ollama.md deleted file mode 100644 index 64bc9324..00000000 --- a/docs/Examples/local_models/smart_scraper_multi_lite_ollama.md +++ /dev/null @@ -1,46 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import json -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/local_models/smart_scraper_multi_ollama.md b/docs/Examples/local_models/smart_scraper_multi_ollama.md deleted file mode 100644 index 2ebbea19..00000000 --- a/docs/Examples/local_models/smart_scraper_multi_ollama.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" - -import json -from scrapegraphai.graphs import SmartScraperMultiGraph - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - - "verbose": True, - "headless": False -} - - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/local_models/smart_scraper_ollama.md b/docs/Examples/local_models/smart_scraper_ollama.md deleted file mode 100644 index 772f2aee..00000000 --- a/docs/Examples/local_models/smart_scraper_ollama.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ -smart_scraper_graph = SmartScraperGraph( - prompt="Find some information about what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/local_models/smart_scraper_schema_ollama.md b/docs/Examples/local_models/smart_scraper_schema_ollama.md deleted file mode 100644 index 1ee9cf53..00000000 --- a/docs/Examples/local_models/smart_scraper_schema_ollama.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with schema -""" -import json -from typing import List -from pydantic import BaseModel, Field -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -graph_config = { - "llm": { - "model": "ollama/llama3.1", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, - "headless": False -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/local_models/xml_scraper_graph_multi_ollama.md b/docs/Examples/local_models/xml_scraper_graph_multi_ollama.md deleted file mode 100644 index e4df1cd1..00000000 --- a/docs/Examples/local_models/xml_scraper_graph_multi_ollama.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" - -import os -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - - "verbose": True, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/local_models/xml_scraper_ollama.md b/docs/Examples/local_models/xml_scraper_ollama.md deleted file mode 100644 index 353c4465..00000000 --- a/docs/Examples/local_models/xml_scraper_ollama.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "ollama/llama3", - "temperature": 0, - # "model_tokens": 2000, # set context length arbitrarily - "base_url": "http://localhost:11434", - }, - "verbose": True, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/mistral/document_scraper_mistral.md b/docs/Examples/mistral/document_scraper_mistral.md deleted file mode 100644 index 5434ce0a..00000000 --- a/docs/Examples/mistral/document_scraper_mistral.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/mistral/smart_scraper_lite_mistral.md b/docs/Examples/mistral/smart_scraper_lite_mistral.md deleted file mode 100644 index 82b62f38..00000000 --- a/docs/Examples/mistral/smart_scraper_lite_mistral.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("MISTRAL_API_KEY"), - "model": "mistral/mistral-medium", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/mistral/smart_scraper_multi_lite_mistral.md b/docs/Examples/mistral/smart_scraper_multi_lite_mistral.md deleted file mode 100644 index eca9b597..00000000 --- a/docs/Examples/mistral/smart_scraper_multi_lite_mistral.md +++ /dev/null @@ -1,36 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("MISTRAL_API_KEY"), - "model": "mistral/mistral-medium", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/model_instance/smart_scraper_with_model_instace.md b/docs/Examples/model_instance/smart_scraper_with_model_instace.md deleted file mode 100644 index 27edd420..00000000 --- a/docs/Examples/model_instance/smart_scraper_with_model_instace.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper and model_instace -""" - -import os, json -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info -from langchain_community.chat_models.moonshot import MoonshotChat -from dotenv import load_dotenv -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -llm_instance_config = { - "model": "moonshot-v1-8k", - "base_url": "https://api.moonshot.cn/v1", - "moonshot_api_key": os.getenv("MOONLIGHT_API_KEY"), -} - - -llm_model_instance = MoonshotChat(**llm_instance_config) - -graph_config = { - "llm": { - "model_instance": llm_model_instance, - "model_tokens": 10000 - }, - "verbose": True, - "headless": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/moonshot/code_generator_graph_moonshot.md b/docs/Examples/moonshot/code_generator_graph_moonshot.md deleted file mode 100644 index be9dcbd7..00000000 --- a/docs/Examples/moonshot/code_generator_graph_moonshot.md +++ /dev/null @@ -1,68 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" - -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from langchain_community.chat_models.moonshot import MoonshotChat -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -llm_instance_config = { - "model": "moonshot-v1-8k", - "base_url": "https://api.moonshot.cn/v1", - "moonshot_api_key": os.getenv("MOONLIGHT_API_KEY"), -} - -llm_model_instance = MoonshotChat(**llm_instance_config) - -graph_config = { - "llm": { - "model_instance": llm_model_instance, - "model_tokens": 10000 - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) \ No newline at end of file diff --git a/docs/Examples/moonshot/document_scraper_moonshot.md b/docs/Examples/moonshot/document_scraper_moonshot.md deleted file mode 100644 index 5434ce0a..00000000 --- a/docs/Examples/moonshot/document_scraper_moonshot.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -mistral_key = os.getenv("MISTRAL_API_KEY") - -graph_config = { - "llm": { - "api_key": mistral_key, - "model": "mistralai/open-mistral-nemo", - }, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/moonshot/readme.md b/docs/Examples/moonshot/readme.md deleted file mode 100644 index bbf8d779..00000000 --- a/docs/Examples/moonshot/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -```python -This folder offer an example of how to use ScrapeGraph-AI with Moonshot and SmartScraperGraph. More usage examples can refer to openai exapmles. \ No newline at end of file diff --git a/docs/Examples/moonshot/smart_scraper_lite_moonshot.md b/docs/Examples/moonshot/smart_scraper_lite_moonshot.md deleted file mode 100644 index d9244630..00000000 --- a/docs/Examples/moonshot/smart_scraper_lite_moonshot.md +++ /dev/null @@ -1,32 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/moonshot/smart_scraper_multi_concat_moonshot.md b/docs/Examples/moonshot/smart_scraper_multi_concat_moonshot.md deleted file mode 100644 index 65c6b8dc..00000000 --- a/docs/Examples/moonshot/smart_scraper_multi_concat_moonshot.md +++ /dev/null @@ -1,53 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from langchain_community.chat_models.moonshot import MoonshotChat -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiConcatGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -llm_instance_config = { - "model": "moonshot-v1-8k", - "base_url": "https://api.moonshot.cn/v1", - "moonshot_api_key": os.getenv("MOONLIGHT_API_KEY"), -} - - -llm_model_instance = MoonshotChat(**llm_instance_config) - -graph_config = { - "llm": { - "model_instance": llm_model_instance, - "model_tokens": 10000 - }, - "verbose": True, - "headless": True, -} - - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiConcatGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/moonshot/smart_scraper_multi_lite_moonshot.md b/docs/Examples/moonshot/smart_scraper_multi_lite_moonshot.md deleted file mode 100644 index a27e2432..00000000 --- a/docs/Examples/moonshot/smart_scraper_multi_lite_moonshot.md +++ /dev/null @@ -1,35 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("MOONSHOT_API_KEY"), - "model": "moonshot/moonshot-v1-8b", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/moonshot/smart_scraper_with_moonshot.md b/docs/Examples/moonshot/smart_scraper_with_moonshot.md deleted file mode 100644 index 3d40a28c..00000000 --- a/docs/Examples/moonshot/smart_scraper_with_moonshot.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper and model_instace -""" -import os -import json -from langchain_community.chat_models.moonshot import MoonshotChat -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -llm_instance_config = { - "model": "moonshot-v1-8k", - "base_url": "https://api.moonshot.cn/v1", - "moonshot_api_key": os.getenv("MOONLIGHT_API_KEY"), -} - - -llm_model_instance = MoonshotChat(**llm_instance_config) - -graph_config = { - "llm": { - "model_instance": llm_model_instance, - "model_tokens": 10000 - }, - "verbose": True, - "headless": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me what does the company do, the name and a contact email.", - source="https://scrapegraphai.com/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/nemotron/document_scraper_nemotron.md b/docs/Examples/nemotron/document_scraper_nemotron.md deleted file mode 100644 index eef2ca7c..00000000 --- a/docs/Examples/nemotron/document_scraper_nemotron.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -nemotron_key = os.getenv("NEMOTRON_APIKEY") - -graph_config = { - "llm": { - "api_key": nemotron_key, - "model": "nvidia/meta/llama3-70b-instruct", - }, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/nemotron/smart_scraper_lite_nemotron.md b/docs/Examples/nemotron/smart_scraper_lite_nemotron.md deleted file mode 100644 index d6388bf0..00000000 --- a/docs/Examples/nemotron/smart_scraper_lite_nemotron.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_API_KEY"), - "model": "nemotron/nemotron-3.5-turbo", - "base_url": "http://127.0.0.1:3000/v1", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/nemotron/smart_scraper_multi_lite_nemotron.md b/docs/Examples/nemotron/smart_scraper_multi_lite_nemotron.md deleted file mode 100644 index 42a3b088..00000000 --- a/docs/Examples/nemotron/smart_scraper_multi_lite_nemotron.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": os.getenv("NEMOTRON_API_KEY"), - "model": "nemotron/nemotron-3-8b-chat", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/oneapi/document_scraper_oneapi.md b/docs/Examples/oneapi/document_scraper_oneapi.md deleted file mode 100644 index 2260fed6..00000000 --- a/docs/Examples/oneapi/document_scraper_oneapi.md +++ /dev/null @@ -1,43 +0,0 @@ -```python -""" -document_scraper example -""" -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "api_key": "***************************", - "model": "oneapi/qwen-turbo", - "base_url": "http://127.0.0.1:3000/v1", # ่ฎพ็ฝฎ OneAPI URL - } -} - - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/oneapi/smart_scraper_lite_oneapi.md b/docs/Examples/oneapi/smart_scraper_lite_oneapi.md deleted file mode 100644 index c45ac243..00000000 --- a/docs/Examples/oneapi/smart_scraper_lite_oneapi.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("ONEAPI_API_KEY"), - "model": "oneapi/gpt-3.5-turbo", - "base_url": "http://127.0.0.1:3000/v1", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/oneapi/smart_scraper_multi_lite_oneapi.md b/docs/Examples/oneapi/smart_scraper_multi_lite_oneapi.md deleted file mode 100644 index 2152ba4e..00000000 --- a/docs/Examples/oneapi/smart_scraper_multi_lite_oneapi.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/openai/document_scraper_openai.md b/docs/Examples/openai/document_scraper_openai.md deleted file mode 100644 index 30d38e7b..00000000 --- a/docs/Examples/openai/document_scraper_openai.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "openai/gpt-4o", - } -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/openai/result.md b/docs/Examples/openai/result.md deleted file mode 100644 index 958bd5f3..00000000 --- a/docs/Examples/openai/result.md +++ /dev/null @@ -1,3 +0,0 @@ -```python -0,1,2,3 -Rotary Pendulum RL,DQN,Multi Agents HAED,Wireless ESC for Modular Drones diff --git a/docs/Examples/openai/smart_scraper_lite_openai.md b/docs/Examples/openai/smart_scraper_lite_openai.md deleted file mode 100644 index 737c756b..00000000 --- a/docs/Examples/openai/smart_scraper_lite_openai.md +++ /dev/null @@ -1,33 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -smart_scraper_lite_graph = SmartScraperLiteGraph( - prompt="Who is Marco Perini?", - source="https://perinim.github.io/", - config=graph_config -) - -result = smart_scraper_lite_graph.run() -print(json.dumps(result, indent=4)) - -graph_exec_info = smart_scraper_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - diff --git a/docs/Examples/openai/smart_scraper_multi_lite_openai.md b/docs/Examples/openai/smart_scraper_multi_lite_openai.md deleted file mode 100644 index c8f7ef66..00000000 --- a/docs/Examples/openai/smart_scraper_multi_lite_openai.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "api_key": os.getenv("OPENAI_API_KEY"), - "model": "openai/gpt-4o", - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/scrapegraph-api/smart_scraper_api.md b/docs/Examples/scrapegraph-api/smart_scraper_api.md deleted file mode 100644 index ab9ad6a9..00000000 --- a/docs/Examples/scrapegraph-api/smart_scraper_api.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -graph_config = { - "llm": { - "model": "scrapegraphai/smart-scraper", - "api_key": os.getenv("SCRAPEGRAPH_API_KEY") - }, - "verbose": True, - "headless": False, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="Extract me all the articles", - source="https://www.wired.com", - config=graph_config -) - -result = smart_scraper_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/single_node/fetch_node.md b/docs/Examples/single_node/fetch_node.md deleted file mode 100644 index 133c56bc..00000000 --- a/docs/Examples/single_node/fetch_node.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" - -from scrapegraphai.nodes import FetchNode - -# ************************************************ -# Define the node -# ************************************************ - - -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], - node_config={ - "headless": False - } -) - -# ************************************************ -# Test the node -# ************************************************ - -state = { - "url": "https://twitter.com/home" -} - -result = fetch_node.execute(state) - -print(result) diff --git a/docs/Examples/single_node/image2text_node.md b/docs/Examples/single_node/image2text_node.md deleted file mode 100644 index 6e98b34d..00000000 --- a/docs/Examples/single_node/image2text_node.md +++ /dev/null @@ -1,55 +0,0 @@ -```python -""" -Example of ImageToTextNode -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.nodes import ImageToTextNode -from scrapegraphai.models import OpenAIImageToText - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "gpt-4o", - "temperature": 0, - }, -} - -# ************************************************ -# Define the node -# ************************************************ - -llm_model = OpenAIImageToText(graph_config["llm"]) - -image_to_text_node = ImageToTextNode( - input="img_url", - output=["img_desc"], - node_config={ - "llm_model": llm_model, - "headless": False - } -) - -# ************************************************ -# Test the node -# ************************************************ - -state = { - "img_url": [ - "https://perinim.github.io/assets/img/rotary_pybullet.jpg", - "https://perinim.github.io/assets/img/value-policy-heatmaps.jpg", - ], -} - -result = image_to_text_node.execute(state) - -print(result) diff --git a/docs/Examples/single_node/kg_node.md b/docs/Examples/single_node/kg_node.md deleted file mode 100644 index f1111bad..00000000 --- a/docs/Examples/single_node/kg_node.md +++ /dev/null @@ -1,80 +0,0 @@ -```python -""" -Example of knowledge graph node -""" - -import os -from langchain_openai import ChatOpenAI -from scrapegraphai.nodes import KnowledgeGraphNode - -job_postings = { - "Job Postings": { - "Company A": [ - { - "title": "Software Engineer", - "description": "Develop and maintain software applications.", - "location": "New York, NY", - "date_posted": "2024-05-01", - "requirements": ["Python", "Django", "REST APIs"] - }, - { - "title": "Data Scientist", - "description": "Analyze and interpret complex data.", - "location": "San Francisco, CA", - "date_posted": "2024-05-05", - "requirements": ["Python", "Machine Learning", "SQL"] - } - ], - "Company B": [ - { - "title": "Project Manager", - "description": "Manage software development projects.", - "location": "Boston, MA", - "date_posted": "2024-04-20", - "requirements": ["Project Management", "Agile", "Scrum"] - } - ] - } -} - - - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { - "llm": { - "api_key": openai_key, - "model": "gpt-4o", - "temperature": 0, - }, - "verbose": True, -} - -# ************************************************ -# Define the node -# ************************************************ - -llm_model = ChatOpenAI(graph_config["llm"]) - -robots_node = KnowledgeGraphNode( - input="user_prompt & answer_dict", - output=["is_scrapable"], - node_config={"llm_model": llm_model} -) - -# ************************************************ -# Test the node -# ************************************************ - -state = { - "user_prompt": "What are the job postings?", - "answer_dict": job_postings -} - -result = robots_node.execute(state) - -print(result) diff --git a/docs/Examples/single_node/robot_node.md b/docs/Examples/single_node/robot_node.md deleted file mode 100644 index a80ac81a..00000000 --- a/docs/Examples/single_node/robot_node.md +++ /dev/null @@ -1,50 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" - -from langchain_community.chat_models import ChatOllama -from scrapegraphai.nodes import RobotsNode - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "llama3", - "temperature": 0, - "streaming": True - }, - "embeddings": { - "model": "nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - } -} - -# ************************************************ -# Define the node -# ************************************************ - -llm_model = ChatOllama(graph_config["llm"]) - -robots_node = RobotsNode( - input="url", - output=["is_scrapable"], - node_config={"llm_model": llm_model, - "headless": False - } -) - -# ************************************************ -# Test the node -# ************************************************ - -state = { - "url": "https://twitter.com/home" -} - -result = robots_node.execute(state) - -print(result) diff --git a/docs/Examples/single_node/search_internet_node.md b/docs/Examples/single_node/search_internet_node.md deleted file mode 100644 index b2fd3dad..00000000 --- a/docs/Examples/single_node/search_internet_node.md +++ /dev/null @@ -1,51 +0,0 @@ -```python -""" -Example of custom graph using existing nodes -""" - -from langchain_community.chat_models import ChatOllama -from scrapegraphai.nodes import SearchInternetNode - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "llama3", - "temperature": 0, - "streaming": True - }, - "search_engine": "google", - "max_results": 3, - "verbose": True -} - -# ************************************************ -# Define the node -# ************************************************ - -llm_model = ChatOllama(graph_config["llm"]) - -search_node = SearchInternetNode( - input="user_input", - output=["search_results"], - node_config={ - "llm_model": llm_model, - "search_engine": graph_config["search_engine"], - "max_results": graph_config["max_results"], - "verbose": graph_config["verbose"] - } -) - -# ************************************************ -# Test the node -# ************************************************ - -state = { - "user_input": "What is the capital of France?" -} - -result = search_node.execute(state) - -print(result) diff --git a/docs/Examples/together/code_generator_graph_togehter.md b/docs/Examples/together/code_generator_graph_togehter.md deleted file mode 100644 index 61f10b95..00000000 --- a/docs/Examples/together/code_generator_graph_togehter.md +++ /dev/null @@ -1,61 +0,0 @@ -```python -""" -Basic example of scraping pipeline using Code Generator with schema -""" - -import os, json -from typing import List -from dotenv import load_dotenv -from pydantic import BaseModel, Field -from scrapegraphai.graphs import CodeGeneratorGraph - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_KEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, - "headless": False, - "reduction": 2, - "max_iterations": { - "overall": 10, - "syntax": 3, - "execution": 3, - "validation": 3, - "semantic": 3 - }, - "output_file_name": "extracted_data.py" -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -code_generator_graph = CodeGeneratorGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = code_generator_graph.run() -print(result) diff --git a/docs/Examples/together/csv_scraper_graph_multi_together.md b/docs/Examples/together/csv_scraper_graph_multi_together.md deleted file mode 100644 index a5a22e9a..00000000 --- a/docs/Examples/together/csv_scraper_graph_multi_together.md +++ /dev/null @@ -1,57 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperMultiGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} -# ************************************************ -# Create the CSVScraperMultiGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperMultiGraph( - prompt="List me all the last names", - source=[str(text), str(text)], - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/together/csv_scraper_together.md b/docs/Examples/together/csv_scraper_together.md deleted file mode 100644 index 09ff8783..00000000 --- a/docs/Examples/together/csv_scraper_together.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using CSVScraperGraph from CSV documents -""" -import os -from dotenv import load_dotenv -import pandas as pd -from scrapegraphai.graphs import CSVScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the CSV file -# ************************************************ - -FILE_NAME = "inputs/username.csv" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -text = pd.read_csv(file_path) - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} -# ************************************************ -# Create the CSVScraperGraph instance and run it -# ************************************************ - -csv_scraper_graph = CSVScraperGraph( - prompt="List me all the last names", - source=str(text), # Pass the content of the file, not the file object - config=graph_config -) - -result = csv_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = csv_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/together/depth_search_graph_together.md b/docs/Examples/together/depth_search_graph_together.md deleted file mode 100644 index b7c679f4..00000000 --- a/docs/Examples/together/depth_search_graph_together.md +++ /dev/null @@ -1,31 +0,0 @@ -```python -""" -depth_search_graph_opeani example -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import DepthSearchGraph - -load_dotenv() - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, - "headless": False, - "depth": 2, - "only_inside_links": False, -} - -search_graph = DepthSearchGraph( - prompt="List me all the projects with their description", - source="https://perinim.github.io", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/together/document_scraper_together.md b/docs/Examples/together/document_scraper_together.md deleted file mode 100644 index 568dd596..00000000 --- a/docs/Examples/together/document_scraper_together.md +++ /dev/null @@ -1,40 +0,0 @@ -```python -""" -document_scraper example -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import DocumentScraperGraph - -load_dotenv() - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -source = """ - The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian - circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. - Divided into three major sectionsโ€”Inferno, Purgatorio, and Paradisoโ€”the narrative traces the journey of Dante - from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. - Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood - through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided - by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, - the Beatrice of his earlier poetry, through the celestial spheres of Paradise. -""" - -pdf_scraper_graph = DocumentScraperGraph( - prompt="Summarize the text and find the main topics", - source=source, - config=graph_config, -) -result = pdf_scraper_graph.run() - -print(json.dumps(result, indent=4)) \ No newline at end of file diff --git a/docs/Examples/together/json_scraper_multi_together.md b/docs/Examples/together/json_scraper_multi_together.md deleted file mode 100644 index c504bab8..00000000 --- a/docs/Examples/together/json_scraper_multi_together.md +++ /dev/null @@ -1,39 +0,0 @@ -```python -""" -Module for showing how JSONScraperMultiGraph multi works -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperMultiGraph - -load_dotenv() - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -sources = [text, text] - -multiple_search_graph = JSONScraperMultiGraph( - prompt= "List me all the authors, title and genres of the books", - source= sources, - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/together/json_scraper_together.md b/docs/Examples/together/json_scraper_together.md deleted file mode 100644 index abe2ec33..00000000 --- a/docs/Examples/together/json_scraper_together.md +++ /dev/null @@ -1,58 +0,0 @@ -```python -""" -Basic example of scraping pipeline using JSONScraperGraph from JSON documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import JSONScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the JSON file -# ************************************************ -together_key = os.getenv("TOGETHER_APIKEY") - -FILE_NAME = "inputs/example.json" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the JSONScraperGraph instance and run it -# ************************************************ - -json_scraper_graph = JSONScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = json_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = json_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/together/rate_limit_together.md b/docs/Examples/together/rate_limit_together.md deleted file mode 100644 index 268e8b19..00000000 --- a/docs/Examples/together/rate_limit_together.md +++ /dev/null @@ -1,48 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper with a custom rate limit -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - "rate_limit": { - "requests_per_second": 1 - } - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/scrape_plain_text_together.md b/docs/Examples/together/scrape_plain_text_together.md deleted file mode 100644 index 982c4e3f..00000000 --- a/docs/Examples/together/scrape_plain_text_together.md +++ /dev/null @@ -1,56 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper from text -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the text file -# ************************************************ - -FILE_NAME = "inputs/plain_html_example.txt" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -# It could be also a http request using the request model -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - source=text, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/script_generator_together.md b/docs/Examples/together/script_generator_together.md deleted file mode 100644 index ca4020bd..00000000 --- a/docs/Examples/together/script_generator_together.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/script_multi_generator_together.md b/docs/Examples/together/script_multi_generator_together.md deleted file mode 100644 index 51ddc3e3..00000000 --- a/docs/Examples/together/script_multi_generator_together.md +++ /dev/null @@ -1,54 +0,0 @@ -```python -""" -Basic example of scraping pipeline using ScriptCreatorGraph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import ScriptCreatorMultiGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "library": "beautifulsoup" -} - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -urls=[ - "https://schultzbergagency.com/emil-raste-karlsen/", - "https://schultzbergagency.com/johanna-hedberg/", -] - -# ************************************************ -# Create the ScriptCreatorGraph instance and run it -# ************************************************ - -script_creator_graph = ScriptCreatorMultiGraph( - prompt="Find information about actors", - # also accepts a string with the already downloaded HTML code - source=urls, - config=graph_config -) - -result = script_creator_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = script_creator_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/search_graph_schema_together.md b/docs/Examples/together/search_graph_schema_together.md deleted file mode 100644 index 7783ac10..00000000 --- a/docs/Examples/together/search_graph_schema_together.md +++ /dev/null @@ -1,62 +0,0 @@ -```python -""" -Example of Search Graph -""" - -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Dish(BaseModel): - name: str = Field(description="The name of the dish") - description: str = Field(description="The description of the dish") - -class Dishes(BaseModel): - dishes: List[Dish] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me Chioggia's famous dishes", - config=graph_config, - schema=Dishes -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/together/search_graph_together.md b/docs/Examples/together/search_graph_together.md deleted file mode 100644 index 976d996b..00000000 --- a/docs/Examples/together/search_graph_together.md +++ /dev/null @@ -1,35 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "max_results": 2, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) diff --git a/docs/Examples/together/search_link_graph_together.md b/docs/Examples/together/search_link_graph_together.md deleted file mode 100644 index 3f0a78f0..00000000 --- a/docs/Examples/together/search_link_graph_together.md +++ /dev/null @@ -1,47 +0,0 @@ -```python -""" -Example of Search Graph -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SearchGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -load_dotenv() - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SearchGraph instance and run it -# ************************************************ - -search_graph = SearchGraph( - prompt="List me the best escursions near Trento", - config=graph_config -) - -result = search_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = search_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json and csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/together/smart_scraper_lite_together.md b/docs/Examples/together/smart_scraper_lite_together.md deleted file mode 100644 index 3e7b89a1..00000000 --- a/docs/Examples/together/smart_scraper_lite_together.md +++ /dev/null @@ -1,2 +0,0 @@ -```python - \ No newline at end of file diff --git a/docs/Examples/together/smart_scraper_multi_lite_together.md b/docs/Examples/together/smart_scraper_multi_lite_together.md deleted file mode 100644 index 2152ba4e..00000000 --- a/docs/Examples/together/smart_scraper_multi_lite_together.md +++ /dev/null @@ -1,44 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiLiteGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - config=graph_config -) - -result = smart_scraper_multi_lite_graph.run() -print(json.dumps(result, indent=4)) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/smart_scraper_multi_together.md b/docs/Examples/together/smart_scraper_multi_together.md deleted file mode 100644 index 7d43a6a7..00000000 --- a/docs/Examples/together/smart_scraper_multi_together.md +++ /dev/null @@ -1,41 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -import json -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperMultiGraph - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ******************************************************* -# Create the SmartScraperMultiGraph instance and run it -# ******************************************************* - -multiple_search_graph = SmartScraperMultiGraph( - prompt="Who is Marco Perini?", - source= [ - "https://perinim.github.io/", - "https://perinim.github.io/cv/" - ], - schema=None, - config=graph_config -) - -result = multiple_search_graph.run() -print(json.dumps(result, indent=4)) diff --git a/docs/Examples/together/smart_scraper_schema_together.md b/docs/Examples/together/smart_scraper_schema_together.md deleted file mode 100644 index 85ff15b9..00000000 --- a/docs/Examples/together/smart_scraper_schema_together.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from typing import List -from pydantic import BaseModel, Field -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the output schema for the graph -# ************************************************ - -class Project(BaseModel): - title: str = Field(description="The title of the project") - description: str = Field(description="The description of the project") - -class Projects(BaseModel): - projects: List[Project] - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - schema=Projects, - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/smart_scraper_together.md b/docs/Examples/together/smart_scraper_together.md deleted file mode 100644 index 145d969c..00000000 --- a/docs/Examples/together/smart_scraper_together.md +++ /dev/null @@ -1,45 +0,0 @@ -```python -""" -Basic example of scraping pipeline using SmartScraper -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description.", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects/", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) diff --git a/docs/Examples/together/xml_scraper_graph_multi_together.md b/docs/Examples/together/xml_scraper_graph_multi_together.md deleted file mode 100644 index bc805559..00000000 --- a/docs/Examples/together/xml_scraper_graph_multi_together.md +++ /dev/null @@ -1,59 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperMultiGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperMultiGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the XMLScraperMultiGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperMultiGraph( - prompt="List me all the authors, title and genres of the books", - source=[text, text], # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Examples/together/xml_scraper_together.md b/docs/Examples/together/xml_scraper_together.md deleted file mode 100644 index ff1f2792..00000000 --- a/docs/Examples/together/xml_scraper_together.md +++ /dev/null @@ -1,60 +0,0 @@ -```python -""" -Basic example of scraping pipeline using XMLScraperGraph from XML documents -""" -import os -from dotenv import load_dotenv -from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info - -load_dotenv() - -# ************************************************ -# Read the XML file -# ************************************************ - -FILE_NAME = "inputs/books.xml" -curr_dir = os.path.dirname(os.path.realpath(__file__)) -file_path = os.path.join(curr_dir, FILE_NAME) - -with open(file_path, 'r', encoding="utf-8") as file: - text = file.read() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - - -together_key = os.getenv("TOGETHER_APIKEY") - -graph_config = { - "llm": { - "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "api_key": together_key, - }, - "verbose": True, -} - -# ************************************************ -# Create the XMLScraperGraph instance and run it -# ************************************************ - -xml_scraper_graph = XMLScraperGraph( - prompt="List me all the authors, title and genres of the books", - source=text, # Pass the content of the file, not the file object - config=graph_config -) - -result = xml_scraper_graph.run() -print(result) - -# ************************************************ -# Get graph execution info -# ************************************************ - -graph_exec_info = xml_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/docs/Graphs/Abstract_graph.md b/docs/Graphs/Abstract_graph.md deleted file mode 100644 index 36c163c1..00000000 --- a/docs/Graphs/Abstract_graph.md +++ /dev/null @@ -1,94 +0,0 @@ -# ๐Ÿ†Ž Abstract Graph - -The `AbstractGraph` module provides a scaffolding class for creating a graph representation and executing it. - -## Classes - -### `AbstractGraph` - -`AbstractGraph` is an abstract base class for creating and executing graph representations. It defines the structure and behavior for a graph-based execution model. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (Optional[str])**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (Optional[str])**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. - -#### Methods - -- **`__init__(self, prompt: str, config: dict, source: Optional[str] = None, schema: Optional[str] = None)`** - - Initializes the graph with the given prompt and configuration. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `config (dict)`: Configuration parameters for the graph. - - `source (Optional[str])`: The source of the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`set_common_params(self, params: dict, overwrite=False)`** - - Passes common parameters to every node in the graph unless otherwise defined. - - **Args**: - - `params (dict)`: Common parameters and their values. - - `overwrite (bool)`: Whether to overwrite existing parameters. - -- **`_set_model_token(self, llm)`** - - Sets the model token based on the type of language model. - -- **`_create_llm(self, llm_config: dict, chat=False) -> object`** - - Creates a large language model instance based on the configuration provided. - - **Args**: - - `llm_config (dict)`: Configuration parameters for the language model. - - `chat (bool)`: Whether the model is used for chat. - - **Returns**: An instance of the language model client. - -- **`_create_default_embedder(self, llm_config=None) -> object`** - - Creates a default embedding model instance based on the chosen LLM model. - - **Returns**: An instance of the embedding model client. - -- **`_create_embedder(self, embedder_config: dict) -> object`** - - Creates an embedding model instance based on the configuration provided. - - **Args**: - - `embedder_config (dict)`: Configuration parameters for the embedding model. - - **Returns**: An instance of the embedding model client. - -- **`get_state(self, key=None) -> dict`** - - Retrieves the final state of the graph. - - **Args**: - - `key (Optional[str])`: The key of the final state to retrieve. - - **Returns**: The final state of the graph. - -- **`get_execution_info(self)`** - - Returns the execution information of the graph. - - **Returns**: The execution information of the graph. - -- **`_create_graph(self)`** - - Abstract method to create a graph representation. - - **Returns**: A graph representation. - -- **`run(self) -> str`** - - Abstract method to execute the graph and return the result. - - **Returns**: The result of executing the graph. - -## Example Usage - -Here is an example of how to use the `AbstractGraph` class: - -```python -from abstract_graph import AbstractGraph - -class MyGraph(AbstractGraph): - def _create_graph(self): - # Implementation of graph creation here - return graph - - def run(self): - # Implementation of graph execution here - return "Result" - -my_graph = MyGraph("Example Graph", {"llm": {"model": "gpt-3.5-turbo"}}, "example_source") -result = my_graph.run() -print(result) diff --git a/docs/Graphs/_category_.json b/docs/Graphs/_category_.json deleted file mode 100644 index 7cb609f0..00000000 --- a/docs/Graphs/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Graphs", - "position": 5, - "link": { - "type": "generated-index", - "description": "Description of the Helpers folder" - } -} \ No newline at end of file diff --git a/docs/Graphs/assets/dept_search.png b/docs/Graphs/assets/dept_search.png deleted file mode 100644 index 6cf02d12..00000000 Binary files a/docs/Graphs/assets/dept_search.png and /dev/null differ diff --git a/docs/Graphs/assets/graph.png b/docs/Graphs/assets/graph.png deleted file mode 100644 index cd39756a..00000000 Binary files a/docs/Graphs/assets/graph.png and /dev/null differ diff --git a/docs/Graphs/base_graph.md b/docs/Graphs/base_graph.md deleted file mode 100644 index 1cf97724..00000000 --- a/docs/Graphs/base_graph.md +++ /dev/null @@ -1,82 +0,0 @@ -# โšพ๏ธ Base Graph - -The `BaseGraph` module provides a class for managing and executing a graph composed of interconnected nodes. - -## Classes - -### `BaseGraph` - -`BaseGraph` manages the execution flow of a graph composed of interconnected nodes. - -#### Attributes - -- **nodes (list)**: A list of node instances that will be part of the graph. -- **edges (list)**: A dictionary representing the directed edges of the graph where each key-value pair corresponds to the from-node and to-node relationship. -- **entry_point (str)**: The name of the entry point node from which the graph execution begins. -- **initial_state (dict)**: The initial state passed to the entry point node. -- **use_burr (bool)**: A flag indicating whether to use BurrBridge for execution. -- **burr_config (dict)**: Configuration parameters for BurrBridge. - -#### Methods - -- **`__init__(self, nodes: list, edges: list, entry_point: str, use_burr: bool = False, burr_config: dict = None)`** - - Initializes the graph with the given nodes, edges, and entry point. - - **Args**: - - `nodes (list)`: An iterable of node instances that will be part of the graph. - - `edges (list)`: An iterable of tuples where each tuple represents a directed edge in the graph, defined by a pair of nodes (from_node, to_node). - - `entry_point (BaseNode)`: The node instance that represents the entry point of the graph. - - `use_burr (bool)`: A flag indicating whether to use BurrBridge for execution. - - `burr_config (dict)`: Configuration parameters for BurrBridge. - - **Raises**: - - `Warning`: If the entry point node is not the first node in the list. - -- **`_create_edges(self, edges: list) -> dict`** - - Helper method to create a dictionary of edges from the given iterable of tuples. - - **Args**: - - `edges (list)`: An iterable of tuples representing the directed edges. - - **Returns**: A dictionary of edges with the from-node as keys and to-node as values. - -- **`_execute_standard(self, initial_state: dict) -> Tuple[dict, list]`** - - Executes the graph by traversing nodes starting from the entry point using the standard method. - - **Args**: - - `initial_state (dict)`: The initial state to pass to the entry point node. - - **Returns**: A tuple containing the final state and a list of execution info. - -- **`execute(self, initial_state: dict) -> Tuple[dict, list]`** - - Executes the graph by either using BurrBridge or the standard method. - - **Args**: - - `initial_state (dict)`: The initial state to pass to the entry point node. - - **Returns**: A tuple containing the final state and a list of execution info. - -## Example Usage - -Here is an example of how to use the `BaseGraph` class: - -```python -from base_graph import BaseGraph - -# Define your nodes and edges -fetch_node = ... -parse_node = ... -rag_node = ... -generate_answer_node = ... - -# Create the graph -graph = BaseGraph( - nodes=[fetch_node, parse_node, rag_node, generate_answer_node], - edges=[ - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - ], - entry_point=fetch_node, - use_burr=True, - burr_config={"app_instance_id": "example-instance"} -) - -# Execute the graph -initial_state = {} -final_state, exec_info = graph.execute(initial_state) - -print(final_state) -print(exec_info) diff --git a/docs/Graphs/code_generator_graph.md b/docs/Graphs/code_generator_graph.md deleted file mode 100644 index 90876e6f..00000000 --- a/docs/Graphs/code_generator_graph.md +++ /dev/null @@ -1,57 +0,0 @@ -# ๐Ÿซ€ CodeGeneratorGraph - -**CodeGeneratorGraph** is responsible for generating Python code that extracts specific data from HTML pages using the BeautifulSoup library. It creates a workflow that consists of interconnected nodes handling different stages of the scraping process, from fetching HTML content to generating Python functions based on user prompts. - -### Attributes - -- `prompt (str)`: The user-defined prompt for guiding the code generation process. -- `source (str)`: The URL or local directory from which the HTML content is sourced. -- `config (dict)`: Configuration options for the graph, such as model settings and execution flags. -- `schema (BaseModel)`: Defines the expected output structure for the generated code. -- `llm_model`: An instance of a language model used for generating answers based on the provided prompt. -- `embedder_model`: An embedding model for processing text data into vector embeddings. -- `verbose (bool)`: Flag indicating whether to display detailed execution logs. -- `headless (bool)`: Specifies whether to run the scraping process in headless mode. -- `reduction (int)`: Defines the level of reduction to apply to the HTML content (can be 0, 1, or 2). -- `max_iterations (dict)`: Maximum number of iterations for each reasoning loop of `GenerateCodeNode`. - -### Methods - -#### `__init__(self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None)` -Initializes the CodeGeneratorGraph with a user-defined prompt, data source, configuration, and schema. - -**Args:** -- `prompt (str)`: The prompt guiding the graph's workflow. -- `source (str)`: The data source (URL or local directory) to scrape from. -- `config (dict)`: Configuration parameters, including model settings. -- `schema (Optional[BaseModel])`: Output schema for validating the generated code. - -#### `_create_graph(self) -> BaseGraph` -Creates and connects the nodes for the web scraping process, constructing a graph that defines the workflow from fetching data to generating Python code. - -**Returns:** -- `BaseGraph`: The constructed graph containing the workflow nodes. - -#### `run(self) -> str` -Executes the graph and returns the generated Python code for data extraction. - -**Returns:** -- `str`: The generated code or an error message if no code was created. - -### Example Usage - -```python -code_gen = CodeGeneratorGraph( - prompt="List me all the attractions in Chioggia.", - source="https://en.wikipedia.org/wiki/Chioggia", - config={"llm": {"model": "openai/gpt-3.5-turbo"}} -) - -result = code_gen.run() -print(result) -``` - -## Flow of the process -![assets/graph.png](assets/graph.png) - -*** \ No newline at end of file diff --git a/docs/Graphs/csv_scraper_graph.md b/docs/Graphs/csv_scraper_graph.md deleted file mode 100644 index 3b7f82ca..00000000 --- a/docs/Graphs/csv_scraper_graph.md +++ /dev/null @@ -1,67 +0,0 @@ -# โšพ๏ธ csv scraper graph - -The `CSVScraperGraph` module provides a class for creating and executing a graph that automates the process of extracting information from web pages using a natural language model. - -## Classes - -### `CSVScraperGraph` - -`CSVScraperGraph` is a comprehensive web scraping tool that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (Optional[str])**: The schema for the graph output. -- **input_key (str)**: The key for the input source (either `csv` or `csv_dir`). -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. -- **loader_kwargs**: Additional keyword arguments for the loader. -- **graph**: The graph of nodes representing the workflow for web scraping. -- **final_state**: The final state of the graph after execution. -- **execution_info**: Information about the execution of the graph. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `CSVScraperGraph` with a prompt, source, and configuration. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph. - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self)`** - - Creates the graph of nodes representing the workflow for web scraping. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `CSVScraperGraph` class: - -```python -from csv_scraper_graph import CSVScraperGraph - -# Define the prompt, source, and configuration -prompt = "Extract information from this CSV." -source = "data.csv" -config = { - "llm": {"model": "gpt-3.5-turbo"}, - "verbose": True, - "headless": False, -} - -# Create the scraper graph -scraper_graph = CSVScraperGraph(prompt, source, config) - -# Run the scraper graph -answer = scraper_graph.run() - -print(answer) diff --git a/docs/Graphs/depth_search_graph.md b/docs/Graphs/depth_search_graph.md deleted file mode 100644 index 6d4f50f6..00000000 --- a/docs/Graphs/depth_search_graph.md +++ /dev/null @@ -1,27 +0,0 @@ -# ๐Ÿธ DepthSearchGraph Module -## Overview - -`DepthSearchGraph` is a script generator pipeline that generates the function `extract_data(html: str) -> dict()` for extracting the wanted information from a HTML page. The code generated is in Python and uses the library BeautifulSoup. It requires a user prompt, a source URL, and an output schema. - -## Attributes - -- `prompt` (str): The prompt for the graph. -- `source` (str): The source of the graph. -- `config` (dict): Configuration parameters for the graph. -- `schema` (BaseModel): The schema for the graph output. -- `llm_model`: An instance of a language model client, configured for generating answers. -- `embedder_model`: An instance of an embedding model client, configured for generating embeddings. -- `verbose` (bool): A flag indicating whether to show print statements during execution. -- `headless` (bool): A flag indicating whether to run the graph in headless mode. -- `library` (str): The library used for web scraping (beautiful soup). - -## Args - -- `prompt` (str): The prompt for the graph. -- `source` (str): The source of the graph. -- `config` (dict): Configuration parameters for the graph. -- `schema` (BaseModel): The schema for the graph output. - -## Flow of the process - -![assets/dept_search.png](assets/dept_search.png) \ No newline at end of file diff --git a/docs/Graphs/json_scraper_graph.md b/docs/Graphs/json_scraper_graph.md deleted file mode 100644 index af66a67b..00000000 --- a/docs/Graphs/json_scraper_graph.md +++ /dev/null @@ -1,66 +0,0 @@ -# ๐Ÿข JSONScraperGraph Module - -The `JSONScraperGraph` module defines a class for creating and executing a graph that automates the process of extracting information from JSON files using a natural language model. - -## Classes - -### `JSONScraperGraph` - -`JSONScraperGraph` defines a scraping pipeline for JSON files. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (str)**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. -- **input_key (str)**: The key for the input source (either `json` or `json_dir`). -- **graph**: The graph of nodes representing the workflow for web scraping. -- **final_state**: The final state of the graph after execution. -- **execution_info**: Information about the execution of the graph. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `JSONScraperGraph` with a prompt, source, and configuration. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph. - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `JSONScraperGraph` class: - -```python -from json_scraper_graph import JSONScraperGraph - -# Define the prompt, source, and configuration -prompt = "List me all the attractions in Chioggia." -source = "data/chioggia.json" -config = { - "llm": {"model": "gpt-3.5-turbo"}, - "verbose": True, - "headless": False, -} - -# Create the scraper graph -json_scraper = JSONScraperGraph(prompt, source, config) - -# Run the scraper graph -result = json_scraper.run() - -print(result) diff --git a/docs/Graphs/omni_scraper_graph.md b/docs/Graphs/omni_scraper_graph.md deleted file mode 100644 index ff9ccb0a..00000000 --- a/docs/Graphs/omni_scraper_graph.md +++ /dev/null @@ -1,68 +0,0 @@ -# ๐Ÿ˜‡ OmniScraperGraph Module - -The `OmniScraperGraph` module defines a class for creating and executing a graph that automates the process of extracting information from web pages using a natural language model. - -## Classes - -### `OmniScraperGraph` - -`OmniScraperGraph` is a scraping pipeline that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (str)**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. -- **max_images (int)**: The maximum number of images to process. -- **input_key (str)**: The key for the input source (either `url` or `local_dir`). -- **graph**: The graph of nodes representing the workflow for web scraping. -- **final_state**: The final state of the graph after execution. -- **execution_info**: Information about the execution of the graph. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `OmniScraperGraph` with a prompt, source, and configuration. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph. - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `OmniScraperGraph` class: - -```python -from omni_scraper_graph import OmniScraperGraph - -# Define the prompt, source, and configuration -prompt = "List me all the attractions in Chioggia and describe their pictures." -source = "https://en.wikipedia.org/wiki/Chioggia" -config = { - "llm": {"model": "gpt-4o"}, - "max_images": 10, - "verbose": True, - "headless": False, -} - -# Create the scraper graph -omni_scraper = OmniScraperGraph(prompt, source, config) - -# Run the scraper graph -result = omni_scraper.run() - -print(result) diff --git a/docs/Graphs/omni_search_graph.md b/docs/Graphs/omni_search_graph.md deleted file mode 100644 index e5f33d56..00000000 --- a/docs/Graphs/omni_search_graph.md +++ /dev/null @@ -1,63 +0,0 @@ -# ๐ŸŒˆ OmniSearchGraph Module - -The `OmniSearchGraph` module defines a class for creating and executing a graph that searches the internet for answers to a given prompt. It combines web scraping and internet searching to generate an answer based on the user's query. - -## Classes - -### `OmniSearchGraph` - -`OmniSearchGraph` is a scraping pipeline that searches the internet for answers to a given prompt. It only requires a user prompt to search the internet and generate an answer. - -#### Attributes - -- **prompt (str)**: The user prompt to search the internet. -- **llm_model (dict)**: The configuration for the language model. -- **embedder_model (dict)**: The configuration for the embedder model. -- **headless (bool)**: A flag to run the browser in headless mode. -- **verbose (bool)**: A flag to display the execution information. -- **model_token (int)**: The token limit for the language model. -- **max_results (int)**: The maximum number of results to return. -- **input_key (str)**: The key for the input source (either `url` or `local_dir`). -- **graph**: The graph of nodes representing the workflow for web scraping and searching. -- **final_state**: The final state of the graph after execution. -- **execution_info**: Information about the execution of the graph. - -#### Methods - -- **`__init__(self, prompt: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `OmniSearchGraph` with a prompt, configuration, and schema. - - **Args**: - - `prompt (str)`: The user prompt to search the internet. - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping and searching. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping and searching process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `OmniSearchGraph` class: - -```python -from omni_search_graph import OmniSearchGraph - -# Define the prompt and configuration -prompt = "What is Chioggia famous for?" -config = { - "llm": {"model": "gpt-4o"}, - "max_results": 5, - "headless": False, -} - -# Create the search graph -omni_search = OmniSearchGraph(prompt, config) - -# Run the search graph -result = omni_search.run() - -print(result) diff --git a/docs/Graphs/scrape_graph.md b/docs/Graphs/scrape_graph.md deleted file mode 100644 index cc36369e..00000000 --- a/docs/Graphs/scrape_graph.md +++ /dev/null @@ -1,34 +0,0 @@ -# ๐Ÿ•ท๏ธ ScrapeGraph Module - -The `ScrapeGraph` module defines a class for creating and executing a graph that scrapes a single URL and generates an answer to a given prompt. - -## Classes - -### `ScrapeGraph` - -`ScrapeGraph` is a scraping pipeline that scrapes a single URL and generates an answer to a given prompt. It requires a user prompt and a source URL. - -#### Attributes - -- **prompt (str)**: The user prompt to search the internet. -- **source (str)**: The URL to scrape. -- **config (dict)**: Configuration parameters for the graph. -- **schema (Optional[str])**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `ScrapeGraph` with a prompt, source URL, configuration, and schema. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping and answer generation. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping and answer generation process. - - **Returns**: The answer to the prompt. - diff --git a/docs/Graphs/script_creator_graph.md b/docs/Graphs/script_creator_graph.md deleted file mode 100644 index 39817823..00000000 --- a/docs/Graphs/script_creator_graph.md +++ /dev/null @@ -1,63 +0,0 @@ -# ๐Ÿฆท ScriptCreatorGraph Module - -The `ScriptCreatorGraph` module defines a class for creating and executing a graph that generates web scraping scripts. - -## Classes - -### `ScriptCreatorGraph` - -`ScriptCreatorGraph` defines a scraping pipeline for generating web scraping scripts. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (str)**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. -- **model_token (int)**: The token limit for the language model. -- **library (str)**: The library used for web scraping. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `ScriptCreatorGraph` with a prompt, source, configuration, and schema. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph. - - `config (dict)`: Configuration parameters for the graph. - - `schema (str)`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for generating web scraping scripts. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `ScriptCreatorGraph` class: - -```python -from script_creator_graph import ScriptCreatorGraph - -# Define the prompt, source, and configuration -prompt = "List me all the attractions in Chioggia." -source = "https://en.wikipedia.org/wiki/Chioggia" -config = { - "llm": {"model": "gpt-3.5-turbo"}, - "library": "BeautifulSoup" -} - -# Create the script creator graph -script_creator = ScriptCreatorGraph(prompt, source, config) - -# Run the script creator graph -result = script_creator.run() - -print(result) diff --git a/docs/Graphs/serch_graph.md b/docs/Graphs/serch_graph.md deleted file mode 100644 index a892adb7..00000000 --- a/docs/Graphs/serch_graph.md +++ /dev/null @@ -1,56 +0,0 @@ -# ๐Ÿ˜ฝ SearchGraph Module - -The `SearchGraph` module defines a class for creating and executing a graph that searches the internet for answers to a given prompt. - -## Classes - -### `SearchGraph` - -`SearchGraph` is a scraping pipeline that searches the internet for answers to a given prompt. It only requires a user prompt to search the internet and generate an answer. - -#### Attributes - -- **prompt (str)**: The user prompt to search the internet. -- **llm_model (dict)**: The configuration for the language model. -- **embedder_model (dict)**: The configuration for the embedder model. -- **headless (bool)**: A flag to run the browser in headless mode. -- **verbose (bool)**: A flag to display the execution information. -- **model_token (int)**: The token limit for the language model. - -#### Methods - -- **`__init__(self, prompt: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `SearchGraph` with a prompt, configuration, and schema. - - **Args**: - - `prompt (str)`: The user prompt to search the internet. - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping and searching. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping and searching process. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `SearchGraph` class: - -```python -from search_graph import SearchGraph - -# Define the prompt and configuration -prompt = "What is Chioggia famous for?" -config = { - "llm": {"model": "gpt-3.5-turbo"} -} - -# Create the search graph -search_graph = SearchGraph(prompt, config) - -# Run the search graph -result = search_graph.run() - -print(result) diff --git a/docs/Graphs/smart_scraper_graph.md b/docs/Graphs/smart_scraper_graph.md deleted file mode 100644 index 75b0ff64..00000000 --- a/docs/Graphs/smart_scraper_graph.md +++ /dev/null @@ -1,60 +0,0 @@ -# ๐Ÿ‘„ SmartScraperGraph Module - -The `SmartScraperGraph` module defines a class for creating and executing a graph that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts. - -## Classes - -### `SmartScraperGraph` - -`SmartScraperGraph` is a scraping pipeline that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (str)**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `SmartScraperGraph` with a prompt, source, configuration, and schema. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph. - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the scraping process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `SmartScraperGraph` class: - -```python -from smart_scraper_graph import SmartScraperGraph - -# Define the prompt, source, and configuration -prompt = "List me all the attractions in Chioggia." -source = "https://en.wikipedia.org/wiki/Chioggia" -config = { - "llm": {"model": "gpt-3.5-turbo"} -} - -# Create the smart scraper graph -smart_scraper = SmartScraperGraph(prompt, source, config) - -# Run the smart scraper graph -result = smart_scraper.run() - -print(result) diff --git a/docs/Graphs/smart_scraper_multi.md b/docs/Graphs/smart_scraper_multi.md deleted file mode 100644 index 4a29cea3..00000000 --- a/docs/Graphs/smart_scraper_multi.md +++ /dev/null @@ -1,58 +0,0 @@ -# ๐Ÿงš๐Ÿปโ€โ™‚๏ธ SmartScraperMultiGraph Module - -The `SmartScraperMultiGraph` module defines a class for creating and executing a graph that scrapes a list of URLs and generates answers to a given prompt. - -## Classes - -### `SmartScraperMultiGraph` - -`SmartScraperMultiGraph` is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt. It only requires a user prompt and a list of URLs. - -#### Attributes - -- **prompt (str)**: The user prompt to search the internet. -- **llm_model (dict)**: The configuration for the language model. -- **embedder_model (dict)**: The configuration for the embedder model. -- **headless (bool)**: A flag to run the browser in headless mode. -- **verbose (bool)**: A flag to display the execution information. -- **model_token (int)**: The token limit for the language model. - -#### Methods - -- **`__init__(self, prompt: str, source: List[str], config: dict, schema: Optional[str] = None)`** - - Initializes the `SmartScraperMultiGraph` with a prompt, source (list of URLs), configuration, and schema. - - **Args**: - - `prompt (str)`: The user prompt to search the internet. - - `source (List[str])`: The source of the graph (list of URLs). - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping and searching. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping and searching process. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `SmartScraperMultiGraph` class: - -```python -from smart_scraper_multi_graph import SmartScraperMultiGraph - -# Define the prompt, source (list of URLs), and configuration -prompt = "What is Chioggia famous for?" -source = ["https://en.wikipedia.org/wiki/Chioggia", "https://example.com"] -config = { - "llm": {"model": "gpt-3.5-turbo"} -} - -# Create the smart scraper multi graph -smart_scraper_multi = SmartScraperMultiGraph(prompt, source, config) - -# Run the smart scraper multi graph -result = smart_scraper_multi.run() - -print(result) diff --git a/docs/Graphs/smart_scraper_multi_lite_graph.md b/docs/Graphs/smart_scraper_multi_lite_graph.md deleted file mode 100644 index 82239872..00000000 --- a/docs/Graphs/smart_scraper_multi_lite_graph.md +++ /dev/null @@ -1,33 +0,0 @@ -# ๐Ÿงš๐Ÿปโ€โ™‚๏ธ SmartScraperMultiLiteGraph Module - -The `SmartScraperMultiLiteGraph` module defines a class for creating and executing a graph that scrapes multiple URLs and generates answers to a given prompt. This is a lightweight version of the `SmartScraperMultiGraph`. - -## Classes - -### `SmartScraperMultiLiteGraph` - -`SmartScraperMultiLiteGraph` is a scraping pipeline that scrapes multiple URLs and generates answers to a given prompt. It only requires a user prompt and a list of URLs. - -#### Attributes - -- **prompt (str)**: The user prompt to search the internet. -- **source (List[str])**: The list of URLs to scrape. -- **config (dict)**: Configuration parameters for the graph. -- **schema (Optional[str])**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. - -#### Methods - -- **`__init__(self, prompt: str, source: List[str], config: dict, schema: Optional[str] = None)`** - - Initializes the `SmartScraperMultiLiteGraph` with a prompt, source (list of URLs), configuration, and schema. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping and searching. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping and searching process. - - **Returns**: The answer to the prompt. diff --git a/docs/Graphs/speech_summary.md b/docs/Graphs/speech_summary.md deleted file mode 100644 index c97e63ac..00000000 --- a/docs/Graphs/speech_summary.md +++ /dev/null @@ -1,62 +0,0 @@ -# ๐Ÿฆน๐Ÿปโ€โ™€๏ธ SpeechGraph Module - -The `SpeechGraph` module defines a class for creating and executing a graph that scrapes the web, provides an answer to a given prompt, and generates an audio file. - -## Classes - -### `SpeechGraph` - -`SpeechGraph` is a scraping pipeline that scrapes the web, provides an answer to a given prompt, and generates an audio file. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (str)**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. -- **model_token (int)**: The token limit for the language model. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `SpeechGraph` with a prompt, source (URL or local directory), configuration, and schema. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph (URL or local directory). - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping and audio generation. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the scraping process, generates the answer to the prompt, and saves the audio file. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `SpeechGraph` class: - -```python -from speech_graph import SpeechGraph - -# Define the prompt, source (URL or local directory), and configuration -prompt = "List me all the attractions in Chioggia and generate an audio summary." -source = "https://en.wikipedia.org/wiki/Chioggia" -config = { - "llm": {"model": "gpt-3.5-turbo"}, - "tts_model": {"engine": "Google", "language": "en"} -} - -# Create the speech graph -speech_graph = SpeechGraph(prompt, source, config) - -# Run the speech graph -result = speech_graph.run() - -print(result) diff --git a/docs/Graphs/xml_scraper_graph.md b/docs/Graphs/xml_scraper_graph.md deleted file mode 100644 index 09ad2d37..00000000 --- a/docs/Graphs/xml_scraper_graph.md +++ /dev/null @@ -1,61 +0,0 @@ -# ๐Ÿฐ XMLScraperGraph Module - -The `XMLScraperGraph` module defines a class for creating and executing a graph that extracts information from XML files using a natural language model to interpret and answer prompts. - -## Classes - -### `XMLScraperGraph` - -`XMLScraperGraph` is a scraping pipeline that extracts information from XML files using a natural language model to interpret and answer prompts. - -#### Attributes - -- **prompt (str)**: The prompt for the graph. -- **source (str)**: The source of the graph. -- **config (dict)**: Configuration parameters for the graph. -- **schema (str)**: The schema for the graph output. -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **headless (bool)**: A flag indicating whether to run the graph in headless mode. -- **model_token (int)**: The token limit for the language model. - -#### Methods - -- **`__init__(self, prompt: str, source: str, config: dict, schema: Optional[str] = None)`** - - Initializes the `XMLScraperGraph` with a prompt, source (XML file or directory), configuration, and schema. - - **Args**: - - `prompt (str)`: The prompt for the graph. - - `source (str)`: The source of the graph (XML file or directory). - - `config (dict)`: Configuration parameters for the graph. - - `schema (Optional[str])`: The schema for the graph output. - -- **`_create_graph(self) -> BaseGraph`** - - Creates the graph of nodes representing the workflow for web scraping. - - **Returns**: An instance of `BaseGraph`. - -- **`run(self) -> str`** - - Executes the web scraping process and returns the answer to the prompt. - - **Returns**: The answer to the prompt. - -## Example Usage - -Here is an example of how to use the `XMLScraperGraph` class: - -```python -from xml_scraper_graph import XMLScraperGraph - -# Define the prompt, source (XML file or directory), and configuration -prompt = "List me all the attractions in Chioggia." -source = "data/chioggia.xml" -config = { - "llm": {"model": "gpt-3.5-turbo"} -} - -# Create the XML scraper graph -xml_scraper = XMLScraperGraph(prompt, source, config) - -# Run the XML scraper graph -result = xml_scraper.run() - -print(result) diff --git a/docs/Helpers/_category_.json b/docs/Helpers/_category_.json deleted file mode 100644 index 641352c5..00000000 --- a/docs/Helpers/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Helpers", - "position": 9, - "link": { - "type": "generated-index", - "description": "Description of the Helpers folder" - } -} \ No newline at end of file diff --git a/docs/Helpers/models_tokens.md b/docs/Helpers/models_tokens.md deleted file mode 100644 index d85955d2..00000000 --- a/docs/Helpers/models_tokens.md +++ /dev/null @@ -1,24 +0,0 @@ -# ๐Ÿ™ƒ Models tokens -## Description -Class for describing the cost in tokens for each model - -## Implementation -```python -""" -Models token -""" -models_tokens = { - "gpt-3.5-turbo-0125": 16385, - "gpt-3.5-turbo": 4096, - "gpt-3.5-turbo-1106": 16385, - "gpt-3.5-turbo-instruct": 4096, - "gpt-4-0125-preview": 128000, - "gpt-4-turbo-preview": 128000, - "gpt-4-1106-preview": 128000, - "gpt-4-vision-preview": 128000, - "gpt-4": 8192, - "gpt-4-0613": 8192, - "gpt-4-32k": 32768, - "gpt-4-32k-0613": 32768, -} -``` \ No newline at end of file diff --git a/docs/Helpers/nodes_metadata.md b/docs/Helpers/nodes_metadata.md deleted file mode 100644 index 73fd5245..00000000 --- a/docs/Helpers/nodes_metadata.md +++ /dev/null @@ -1,90 +0,0 @@ -# ๐Ÿฅ‘ Nodes metadata -## Description -Metadata for describing the scrapegraphai package - -## Implementation -```python -""" -Nodes metadata for the scrapegraphai package. -""" - -nodes_metadata = { - "FetchHTMLNode": { - "description": "Fetches HTML content from a given URL.", - "type": "node", - "args": { - "url": "The URL from which to fetch HTML content." - }, - "returns": "Updated state with fetched HTML content under 'document' key." - }, - "GetProbableTagsNode": { - "description": "Identifies probable HTML tags from a document based on a user's question.", - "type": "node", - "args": { - "user_input": "User's query or question.", - "document": "HTML content as a string." - }, - "returns": "Updated state with probable HTML tags under 'tags' key." - }, - "ParseHTMLNode": { - "description": "Parses HTML content to extract specific data.", - "type": "node", - "args": { - "document": "HTML content as a string.", - "tags": "List of HTML tags to focus on during parsing." - }, - "returns": "Updated state with extracted data under 'parsed_document' key." - }, - "RAGNode": { - "description": """A node responsible for reducing the amount of text to be processed - by identifying and retrieving the most relevant chunks of text based on the user's query. - Utilizes RecursiveCharacterTextSplitter for chunking, Html2TextTransformer for HTML to text - conversion, and a combination of FAISS and OpenAIEmbeddings - for efficient information retrieval.""", - "type": "node", - "args": { - "user_input": "The user's query or question guiding the retrieval.", - "document": "The HTML content to be processed and compressed." - }, - "returns": """Updated state with 'relevant_chunks' key containing - the most relevant text chunks.""" - }, - "GenerateAnswerNode": { - "description": "Generates an answer based on the user's input and parsed document.", - "type": "node", - "args": { - "user_input": "User's query or question.", - "parsed_document": "Data extracted from the HTML document." - }, - "returns": "Updated state with the answer under 'answer' key." - }, - "ConditionalNode": { - "description": "Decides the next node to execute based on a condition.", - "type": "conditional_node", - "args": { - "key_name": "The key in the state to check for a condition.", - "next_nodes": """A list of two nodes specifying the next node - to execute based on the condition's outcome.""" - }, - "returns": "The name of the next node to execute." - }, - "ImageToTextNode": { - "description": """Converts image content to text by - extracting visual information and interpreting it.""", - "type": "node", - "args": { - "image_data": "Data of the image to be processed." - }, - "returns": "Updated state with the textual description of the image under 'image_text' key." - }, - "TextToSpeechNode": { - "description": """Converts text into spoken words, allow - ing for auditory representation of the text.""", - "type": "node", - "args": { - "text": "The text to be converted into speech." - }, - "returns": "Updated state with the speech audio file or data under 'speech_audio' key." - } -} -``` \ No newline at end of file diff --git a/docs/Helpers/schemas.md b/docs/Helpers/schemas.md deleted file mode 100644 index 4d74cd52..00000000 --- a/docs/Helpers/schemas.md +++ /dev/null @@ -1,72 +0,0 @@ -# ๐Ÿ—ณ๏ธ Schemas - -## Description -Schemas of the configuration - -## Implementation -```python -""" -Schemas representing the configuration of a graph or node in the ScrapeGraphAI library -""" - -graph_schema = { - "name": "ScrapeGraphAI Graph Configuration", - "description": "JSON schema for representing graphs in the ScrapeGraphAI library", - "type": "object", - "properties": { - "nodes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "node_name": { - "type": "string", - "description": "The unique identifier for the node." - }, - "node_type": { - "type": "string", - "description": "The type of node, must be 'node' or 'conditional_node'." - }, - "args": { - "type": "object", - "description": "The arguments required for the node's execution." - }, - "returns": { - "type": "object", - "description": "The return values of the node's execution." - }, - }, - "required": ["node_name", "node_type", "args", "returns"] - } - }, - "edges": { - "type": "array", - "items": { - "type": "object", - "properties": { - "from": { - "type": "string", - "description": "The node_name of the starting node of the edge." - }, - "to": { - "type": "array", - "items": { - "type": "string" - }, - "description": """An array containing the node_names - of the ending nodes of the edge. - If the 'from' node is a conditional node, - this array must contain exactly two node_names.""" - } - }, - "required": ["from", "to"] - } - }, - "entry_point": { - "type": "string", - "description": "The node_name of the entry point node." - } - }, - "required": ["nodes", "edges", "entry_point"] -} -``` \ No newline at end of file diff --git a/docs/Nodes/_category_.json b/docs/Nodes/_category_.json deleted file mode 100644 index 776ab153..00000000 --- a/docs/Nodes/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Nodes", - "position": "3", - "link": { - "type": "generated-index", - "description": "In the standard structure of Scrapegraph-ai, a graph is composed of nodes, each with its own functionality. The principal nodes of this library include: \n\n- base_node: an abstract class from which all nodes derive. \n- conditional_node: creates conditional branches. \n- fetch_html_node: performs HTML requests. \n- generate_answer_node: generates an answer using a language model (LLM) based on the user's input and the content extracted from a webpage. \n- get_probable_tags_node: utilizes a language model to identify probable HTML tags within a document that are likely to contain information relevant to a user's query. \n- image_to_text_node: given an image, retrieves the text. \n- text_to_speech_node: given text, saves it as an .mp3 file." - } -} diff --git a/docs/Nodes/base_node.md b/docs/Nodes/base_node.md deleted file mode 100644 index 4a8d1463..00000000 --- a/docs/Nodes/base_node.md +++ /dev/null @@ -1,75 +0,0 @@ -# ๐Ÿธ BaseNode Module - -The `BaseNode` module provides an abstract base class for nodes in a graph-based workflow, designed to perform specific actions when executed. - -## Classes - -### `BaseNode` - -`BaseNode` is an abstract base class for nodes in a graph-based workflow. - -#### Attributes - -- **node_name (str)**: The unique identifier name for the node. -- **input (str)**: Boolean expression defining the input keys needed from the state. -- **output (List[str])**: List of output keys to be updated in the state. -- **min_input_len (int)**: Minimum required number of input keys. -- **node_config (Optional[dict])**: Additional configuration for the node. -- **logger (logging.Logger)**: The centralized root logger. - -#### Methods - -- **`__init__(self, node_name: str, node_type: str, input: str, output: List[str], min_input_len: int = 1, node_config: Optional[dict] = None)`** - - Initializes the `BaseNode` with a name, type, input specification, output specification, minimum input length, and configuration. - - **Args**: - - `node_name (str)`: Name for identifying the node. - - `node_type (str)`: Type of the node; must be 'node' or 'conditional_node'. - - `input (str)`: Expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `min_input_len (int, optional)`: Minimum required number of input keys; defaults to 1. - - `node_config (Optional[dict], optional)`: Additional configuration for the node; defaults to None. - - **Raises**: - - `ValueError`: If `node_type` is not one of the allowed types. - -- **`execute(self, state: dict) -> dict`** - - Execute the node's logic based on the current state and update it accordingly. - - **Args**: - - `state (dict)`: The current state of the graph. - - **Returns**: - - `dict`: The updated state after executing the node's logic. - -- **`update_config(self, params: dict, overwrite: bool = False)`** - - Updates the node_config dictionary as well as attributes with the same key. - - **Args**: - - `param (dict)`: The dictionary to update node_config with. - - `overwrite (bool)`: Flag indicating if the values of node_config should be overwritten if their value is not None. - -- **`get_input_keys(self, state: dict) -> List[str]`** - - Determines the necessary state keys based on the input specification. - - **Args**: - - `state (dict)`: The current state of the graph used to parse input keys. - - **Returns**: - - `List[str]`: A list of input keys required for node operation. - - **Raises**: - - `ValueError`: If an error occurs in parsing input keys. - -#### Example Usage - -Here is an example of how to use the `BaseNode` class: - -```python -from base_node import BaseNode - -# Define a custom node class -class MyNode(BaseNode): - def execute(self, state): - # Implementation of node logic here - return state - -# Instantiate the custom node -my_node = MyNode("ExampleNode", "node", "input_spec", ["output_spec"]) - -# Execute the node -updated_state = my_node.execute({'key': 'value'}) - -print(updated_state) diff --git a/docs/Nodes/conditional_node.md b/docs/Nodes/conditional_node.md deleted file mode 100644 index 8ebd1eb0..00000000 --- a/docs/Nodes/conditional_node.md +++ /dev/null @@ -1,51 +0,0 @@ -# ๐Ÿท ConditionalNode Module - -The `ConditionalNode` module implements a node that determines the next step in the graph's execution flow based on the presence and content of a specified key in the graph's state. It extends the `BaseNode` class by adding condition-based logic to the execution process. - -## Classes - -### `ConditionalNode` - -`ConditionalNode` is a node that determines the next step in the graph's execution flow based on the presence and content of a specified key in the graph's state. - -#### Attributes - -- **key_name (str)**: The name of the key in the state to check for its presence. - -#### Methods - -- **`__init__(self, key_name: str, node_name="ConditionalNode")`** - - Initializes the node with the key to check and the next node names based on the condition. - - **Args**: - - `key_name (str)`: The name of the key to check in the graph's state. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "ConditionalNode". - -- **`execute(self, state: dict) -> dict`** - - Checks if the specified key is present in the state and decides the next node accordingly. - - **Args**: - - `state (dict)`: The current state of the graph. - - **Returns**: - - `str`: The name of the next node to execute based on the presence of the key. - -#### Example Usage - -Here is an example of how to use the `ConditionalNode` class: - -```python -from conditional_node import ConditionalNode - -# Define a conditional node to check for the presence of a key -conditional_node = ConditionalNode("example_key") - -# Define the next nodes based on the condition -next_node_true = "NextNodeTrue" -next_node_false = "NextNodeFalse" - -# Execute the conditional node -state = {"example_key": "value"} -state = conditional_node.execute(state) - -# Determine the next node based on the condition -next_node_name = next_node_true if state["next_node"] == 0 else next_node_false - -print(f"Next node to execute: {next_node_name}") diff --git a/docs/Nodes/description_node.md b/docs/Nodes/description_node.md deleted file mode 100644 index aa82153d..00000000 --- a/docs/Nodes/description_node.md +++ /dev/null @@ -1,79 +0,0 @@ -# DescriptionNode - -## Overview - -`DescriptionNode` is a node responsible for compressing the input tokens and storing the document in a vector database for retrieval. Relevant chunks are stored in the state. It allows scraping of big documents without exceeding the token limit of the language model. - -## Attributes - -- `llm_model`: An instance of a language model client, configured for generating answers. -- `verbose` (bool): A flag indicating whether to show print statements during execution. - -## Args - -- `input` (str): Boolean expression defining the input keys needed from the state. -- `output` (List[str]): List of output keys to be updated in the state. -- `node_config` (dict): Additional configuration for the node. -- `node_name` (str): The unique identifier name for the node, defaulting to "DESCRIPTION". - -## Example - -```python -description_node = DescriptionNode( - input="docs", - output=["docs"], - node_config={ - "llm_model": llm_model_instance, - "verbose": True, - "cache_path": "/path/to/cache" - } -) -``` -## Methods - -__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "DESCRIPTION") - -Initializes the DescriptionNode with the given input, output, node configuration, and node name. - -execute(self, state: dict) -> dict - -Executes the node's logic to compress the input tokens and store the document in a vector database for retrieval. - -Args - -state (dict): The current state of the graph. The input keys will be used to fetch the correct data types from the state. -Returns - -dict: The updated state with a new output key containing the compressed documents. -Detailed Implementation - -execute(self, state: dict) -> dict - -Logging: Logs the start of the node execution. -Fetch Documents: Retrieves the documents from the state. -Process Chunks: Iterates over each document chunk and creates a prompt for each chunk using PromptTemplate. -Run Chains: Uses RunnableParallel to run the prompts in parallel and generate summaries. -Update State: Updates the state with the summarized documents. - -## Example Usage - -```python -description_node = DescriptionNode( - input="docs", - output=["docs"], - node_config={ - "llm_model": llm_model_instance, - "verbose": True, - "cache_path": "/path/to/cache" - } -) - -state = { - "docs": [ - {"document": "Document content 1"}, - {"document": "Document content 2"}, - ] -} - -updated_state = description_node.execute(state) -``` \ No newline at end of file diff --git a/docs/Nodes/fetch_html_node.md b/docs/Nodes/fetch_html_node.md deleted file mode 100644 index 3aff12ba..00000000 --- a/docs/Nodes/fetch_html_node.md +++ /dev/null @@ -1,61 +0,0 @@ -# ๐ŸฆŠ FetchNode Module - -The `FetchNode` module implements a node responsible for fetching the HTML content of a specified URL or loading various types of documents (such as JSON, XML, CSV, or PDF) and updating the graph's state with this content. It uses ChromiumLoader to fetch content from web pages asynchronously, ensuring proxy protection. - -## Classes - -### `FetchNode` - -`FetchNode` is a node responsible for fetching the HTML content of a specified URL or loading various types of documents and updating the graph's state with this content. - -#### Attributes - -- **headless (bool)**: A flag indicating whether the browser should run in headless mode. -- **verbose (bool)**: A flag indicating whether to print verbose output during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "Fetch")`** - - Initializes the node with the required input keys and output keys. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (Optional[dict], optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "Fetch". - -- **`execute(self, state)`** - - Executes the node's logic to fetch HTML content from a specified URL or load various types of documents and update the state with this content. - - **Args**: - - `state (dict)`: The current state of the graph. - - **Returns**: - - `dict`: The updated state with a new output key containing the fetched HTML content. - - **Raises**: - - `KeyError`: If the input key is not found in the state, indicating that the necessary information to perform the operation is missing. - -#### Example Usage - -Here is an example of how to use the `FetchNode` class: - -```python -from fetch_node import FetchNode - -# Define a fetch node -fetch_node = FetchNode( - input="url", - output=["fetched_content", "link_urls", "image_urls"] -) - -# Define the state -state = {"url": "https://example.com"} - -# Execute the fetch node -state = fetch_node.execute(state) - -# Retrieve the fetched content and other information from the state -fetched_content = state["fetched_content"] -link_urls = state["link_urls"] -image_urls = state["image_urls"] - -print(f"Fetched Content: {fetched_content}") -print(f"Link URLs: {link_urls}") -print(f"Image URLs: {image_urls}") diff --git a/docs/Nodes/fetch_node_level_k.md b/docs/Nodes/fetch_node_level_k.md deleted file mode 100644 index b7613846..00000000 --- a/docs/Nodes/fetch_node_level_k.md +++ /dev/null @@ -1,50 +0,0 @@ -# FetchNodeLevelK - -## Overview - -`FetchNodeLevelK` is a node responsible for fetching the HTML content of a specified URL and all its sub-links recursively up to a certain level of hyperlink depth. This content is then used to update the graph's state. It uses `ChromiumLoader` to fetch the content from a web page asynchronously (with proxy protection). - -## Attributes - -- `llm_model`: An instance of a language model client, configured for generating answers. -- `verbose` (bool): A flag indicating whether to show print statements during execution. - -## Args - -- `input` (str): Boolean expression defining the input keys needed from the state. -- `output` (List[str]): List of output keys to be updated in the state. -- `node_config` (dict): Additional configuration for the node. -- `node_name` (str): The unique identifier name for the node, defaulting to "FetchLevelK". - -## Example - -```python -fetch_node = FetchNodeLevelK( - input="url", - output=["docs"], - node_config={ - "loader_kwargs": {"some_key": "some_value"}, - "force": False, - "cut": True, - "browser_base": {"api_key": "your_api_key", "project_id": "your_project_id"}, - "depth": 2, - "only_inside_links": False - } -) -``` -## Example Usage - -```python -fetch_node = FetchNodeLevelK( - input="url", - output=["docs"], - node_config={ - "loader_kwargs": {"some_key": "some_value"}, - "force": False, - "cut": True, - "browser_base": {"api_key": "your_api_key", "project_id": "your_project_id"}, - "depth": 2, - "only_inside_links": False - } -) -``` \ No newline at end of file diff --git a/docs/Nodes/generate_answer_csv_node.md b/docs/Nodes/generate_answer_csv_node.md deleted file mode 100644 index ca91961b..00000000 --- a/docs/Nodes/generate_answer_csv_node.md +++ /dev/null @@ -1,58 +0,0 @@ -# ๐Ÿผ GenerateAnswerNodeCSV Module - -The `GenerateAnswerCSVNode` module implements a node responsible for generating an answer using a language model (LLM) based on the user's input and the content extracted from a webpage. It constructs a prompt from the user's input and the scraped content, feeds it to the LLM, and parses the LLM's response to produce an answer. - -## Classes - -### `GenerateAnswerCSVNode` - -`GenerateAnswerCSVNode` is a node responsible for generating an answer using a language model (LLM) based on the user's input and the content extracted from a webpage. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **node_name (str)**: The unique identifier name for the node, defaulting to "GenerateAnswerNodeCsv". -- **node_type (str)**: The type of the node, set to "node" indicating a standard operational node. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "GenerateAnswer")`** - - Initializes the GenerateAnswerNodeCsv with a language model client and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (Optional[dict], optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GenerateAnswerNodeCsv". - -- **`execute(self, state)`** - - Generates an answer by constructing a prompt from the user's input and the scraped content, querying the language model, and parsing its response. - - **Args**: - - `state (dict)`: The current state of the graph, expected to contain 'user_input' and optionally 'parsed_document' or 'relevant_chunks' within 'keys'. - - **Returns**: - - `dict`: The updated state with the 'answer' key containing the generated answer. - - **Raises**: - - `KeyError`: If 'user_input' or 'document' is not found in the state, indicating that the necessary information for generating an answer is missing. - -#### Example Usage - -Here is an example of how to use the `GenerateAnswerCSVNode` class: - -```python -from generate_answer_csv_node import GenerateAnswerCSVNode - -# Define a generate answer node -generate_answer_node = GenerateAnswerCSVNode( - input="user_input & document", - output=["answer"] -) - -# Define the state -state = {"user_input": "What are the attractions in Chioggia?", "document": [document1, document2]} - -# Execute the generate answer node -state = generate_answer_node.execute(state) - -# Retrieve the generated answer from the state -answer = state["answer"] - -print(f"Generated Answer: {answer}") diff --git a/docs/Nodes/generate_answer_node.md b/docs/Nodes/generate_answer_node.md deleted file mode 100644 index ec861d2b..00000000 --- a/docs/Nodes/generate_answer_node.md +++ /dev/null @@ -1,57 +0,0 @@ -# ๐Ÿ” GenerateAnswerNode Module - -The `GenerateAnswerNode` module implements a node responsible for generating an answer using a large language model (LLM) based on the user's input and the content extracted from a webpage. It constructs a prompt from the user's input and the scraped content, feeds it to the LLM, and parses the LLM's response to produce an answer. - -## Classes - -### `GenerateAnswerNode` - -`GenerateAnswerNode` is a node responsible for generating an answer using a large language model (LLM) based on the user's input and the content extracted from a webpage. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "GenerateAnswer")`** - - Initializes the GenerateAnswerNode with a language model client and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GenerateAnswer". - -- **`execute(self, state: dict) -> dict`** - - Generates an answer by constructing a prompt from the user's input and the scraped content, querying the language model, and parsing its response. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the generated answer. - - **Raises**: - - `KeyError`: If the input keys are not found in the state, indicating that the necessary information for generating an answer is missing. - -#### Example Usage - -Here is an example of how to use the `Generaprompt refinedteAnswerNode` class: - -```python -from generate_answer_node import GenerateAnswerNode - -# Define a generate answer node -generate_answer_node = GenerateAnswerNode( - input="user_input & document", - output=["answer"] -) - -# Define the state -state = {"user_input": "What are the attractions in Chioggia?", "document": [document1, document2]} - -# Execute the generate answer node -state = generate_answer_node.execute(state) - -# Retrieve the generated answer from the state -answer = state["answer"] - -print(f"Generated Answer: {answer}") diff --git a/docs/Nodes/generate_answer_node_level_k.md b/docs/Nodes/generate_answer_node_level_k.md deleted file mode 100644 index 2c919af2..00000000 --- a/docs/Nodes/generate_answer_node_level_k.md +++ /dev/null @@ -1,35 +0,0 @@ -# GenerateAnswerNodeKLevel - -## Overview - -`GenerateAnswerNodeKLevel` is a node responsible for compressing the input tokens and storing the document in a vector database for retrieval. Relevant chunks are stored in the state. It allows scraping of big documents without exceeding the token limit of the language model. - -## Attributes - -- `llm_model`: An instance of a language model client, configured for generating answers. -- `verbose` (bool): A flag indicating whether to show print statements during execution. - -## Args - -- `input` (str): Boolean expression defining the input keys needed from the state. -- `output` (List[str]): List of output keys to be updated in the state. -- `node_config` (dict): Additional configuration for the node. -- `node_name` (str): The unique identifier name for the node, defaulting to "GANLK". - -## Example - -```python -generate_answer_node = GenerateAnswerNodeKLevel( - input="vectorial_db", - output=["answer"], - node_config={ - "llm_model": llm_model_instance, - "embedder_model": embedder_model_instance, - "verbose": True, - "force": False, - "script_creator": False, - "is_md_scraper": False, - "additional_info": "Additional information" - } -) -``` \ No newline at end of file diff --git a/docs/Nodes/generate_answer_omni_node.md b/docs/Nodes/generate_answer_omni_node.md deleted file mode 100644 index 41a5c49a..00000000 --- a/docs/Nodes/generate_answer_omni_node.md +++ /dev/null @@ -1,61 +0,0 @@ -# ๐Ÿฏ GenerateAnswerOmniNode Module - -The `GenerateAnswerOmniNode` module implements a node responsible for generating an answer using a large language model (LLM) based on the user's input and the content extracted from a webpage. It constructs a prompt from the user's input and the scraped content, feeds it to the LLM, and parses the LLM's response to produce an answer. - -## Classes - -### `GenerateAnswerOmniNode` - -`GenerateAnswerOmniNode` is a node responsible for generating an answer using a large language model (LLM) based on the user's input and the content extracted from a webpage. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "GenerateAnswerOmni")`** - - Initializes the GenerateAnswerOmniNode with a language model client and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GenerateAnswerOmni". - -- **`execute(self, state: dict) -> dict`** - - Generates an answer by constructing a prompt from the user's input and the scraped content, querying the language model, and parsing its response. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the generated answer. - - **Raises**: - - `KeyError`: If the input keys are not found in the state, indicating that the necessary information for generating an answer is missing. - -#### Example Usage - -Here is an example of how to use the `GenerateAnswerOmniNode` class: - -```python -from generate_answer_node import GenerateAnswerOmniNode - -# Define a generate answer node -generate_answer_omni_node = GenerateAnswerOmniNode( - input="user_input & document & image_description", - output=["answer"] -) - -# Define the state -state = { - "user_input": "What is the history of the Roman Empire?", - "document": [document1, document2], - "image_description": "Image of Roman Colosseum" -} - -# Execute the generate answer node -state = generate_answer_omni_node.execute(state) - -# Retrieve the generated answer from the state -answer = state["answer"] - -print(f"Generated Answer: {answer}") diff --git a/docs/Nodes/generate_code_node.md b/docs/Nodes/generate_code_node.md deleted file mode 100644 index c5d0301c..00000000 --- a/docs/Nodes/generate_code_node.md +++ /dev/null @@ -1,92 +0,0 @@ -# ๐Ÿค– GenerateCodeNode Module - -The `GenerateCodeNode` module dynamically generates Python code for a function named `extract_data(html: str) -> dict()` that extracts specific data from HTML content, adhering to a predefined output schema. It utilizes a sophisticated reasoning loop that iteratively refines the code until it successfully extracts the desired data, ensuring both syntactic correctness and semantic alignment with the user's intent. - -## Classes - -### `GenerateCodeNode` - -`GenerateCodeNode` generates Python code to extract data from HTML based on a schema. The generated code uses the BeautifulSoup library for parsing the HTML. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **output_schema**: The output schema be used for the generated code returned data. -- **max_iterations (dict)**: Maximum number of iterations for each reasoning loop. It should have the following structure: - ```python - { - "overall": 10, # Maximum iterations for the overall reasoning loop - "syntax": 3, # Maximum iterations for the syntax reasoning loop - "execution": 3, # Maximum iterations for the execution reasoning loop - "validation": 3, # Maximum iterations for the validation reasoning loop - "semantic": 3 # Maximum iterations for the semantic comparison loop - } - ``` - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "GenerateCode")`** - - Initializes the `GenerateCodeNode` with the necessary configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GenerateCode". - -- **`execute(self, state: dict) -> dict`** - - Generates Python code that extracts data from HTML based on the specified schema. - - Leverages the analysis from the `PromptRefinerNode` and `HtmlAnalyzerNode` to improve code generation reasoning. - - **Args**: - - `state (dict)`: The current state of the graph, containing user input, refined prompt, HTML analysis, and the reference answer. - - **Returns**: - - `dict`: The updated state with the generated Python code. - - **Raises**: - - `KeyError`: If required input keys are missing from the state. - - `RuntimeError`: If the maximum number of iterations is reached without generating valid code. - -#### Reasoning Loops - -- **`overall_reasoning_loop(self, state: dict) -> dict`** - - Orchestrates the iterative code refinement process, incorporating syntax, execution, validation, and semantic checks. -- **`syntax_reasoning_loop(self, state: dict) -> dict`** - - Iteratively refines the code until it is syntactically correct. -- **`execution_reasoning_loop(self, state: dict) -> dict`** - - Iteratively refines the code until it executes without errors. -- **`validation_reasoning_loop(self, state: dict) -> dict`** - - Iteratively refines the code until its output adheres to the specified schema. -- **`semantic_comparison_loop(self, state: dict) -> dict`** - - Iteratively refines the code until its output semantically aligns with the reference answer. - -**Note:** The `HtmlAnalyzerNode` plays a crucial role in providing context and analysis to the `GenerateCodeNode`, ultimately leading to the generation of more refined and effective code. - -#### Example Usage - -Here is an example of how to use the `GenerateCodeNode` class: - -```python -from generate_code_node import GenerateCodeNode - -# Define a geenrate code node -generate_code_node = GenerateCodeNode( - input="user_prompt & refined_prompt & html_info & reduced_html & answer", - output=["generated_code"] -) - -# Define the state -state = { - "user_prompt": "What are the attractions in Chioggia?", - "refined_prompt": "The user is asking about the attractions in Chioggia.", - "html_info": "The HTML code contains...", - "reduced_html": "...", - "answer": answer -} - -# Execute the generate answer node -state = generate_code_node.execute(state) - -# Retrieve the generated answer from the state -generated_code = state["generated_code"] - -print(f"Code generated: {generated_code}") -``` \ No newline at end of file diff --git a/docs/Nodes/generate_scraper_node.md b/docs/Nodes/generate_scraper_node.md deleted file mode 100644 index 5b2db904..00000000 --- a/docs/Nodes/generate_scraper_node.md +++ /dev/null @@ -1,66 +0,0 @@ -# ๐Ÿญ GenerateScraperNode Module - -The `GenerateScraperNode` module implements a node responsible for generating a Python script for scraping a website using the specified library. It takes the user's prompt and the scraped content as input and generates a Python script that extracts the information requested by the user. - -## Classes - -### `GenerateScraperNode` - -`GenerateScraperNode` is a node responsible for generating a Python script for scraping a website using the specified library. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **library (str)**: The Python library to use for scraping the website. -- **source (str)**: The website to scrape. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], library: str, website: str, node_config: Optional[dict] = None, node_name: str = "GenerateScraper")`** - - Initializes the `GenerateScraperNode` with a language model client, a library, a website, and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `library (str)`: The Python library to use for scraping the website. - - `website (str)`: The website to scrape. - - `node_config (dict, optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GenerateScraper". - -- **`execute(self, state: dict) -> dict`** - - Generates a Python script for scraping a website using the specified library. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the generated answer. - - **Raises**: - - `KeyError`: If input keys are not found in the state, indicating that the necessary information for generating an answer is missing. - -#### Example Usage - -Here is an example of how to use the `GenerateScraperNode` class: - -```python -from generate_scraper_node import GenerateScraperNode - -# Define a generate scraper node -generate_scraper_node = GenerateScraperNode( - input="user_input & document", - output=["generated_script"], - library="requests", - website="https://example.com" -) - -# Define the state -state = { - "user_input": "What information do you want to scrape from the website?", - "document": [document] -} - -# Execute the generate scraper node -state = generate_scraper_node.execute(state) - -# Retrieve the generated script from the state -generated_script = state["generated_script"] - -print(f"Generated Script:\n{generated_script}") diff --git a/docs/Nodes/get_probable_tags_node.md b/docs/Nodes/get_probable_tags_node.md deleted file mode 100644 index b9a70acd..00000000 --- a/docs/Nodes/get_probable_tags_node.md +++ /dev/null @@ -1,63 +0,0 @@ -# ๐Ÿฆ† GetProbableTagsNode Module - -The `GetProbableTagsNode` module implements a node responsible for utilizing a language model to identify probable HTML tags within a document that are likely to contain the information relevant to a user's query. This node generates a prompt describing the task, submits it to the language model, and processes the output to produce a list of probable tags. - -## Classes - -### `GetProbableTagsNode` - -`GetProbableTagsNode` is a node responsible for utilizing a language model to identify probable HTML tags within a document that are likely to contain the information relevant to a user's query. - -#### Attributes - -- **llm_model**: An instance of the language model client used for tag predictions. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: dict, node_name: str = "GetProbableTags")`** - - Initializes the `GetProbableTagsNode` with a language model client and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict)`: Additional configuration for the language model. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GetProbableTags". - -- **`execute(self, state: dict) -> dict`** - - Generates a list of probable HTML tags based on the user's input and updates the state with this list. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data types from the state. - - **Returns**: - - `dict`: The updated state with the input key containing a list of probable HTML tags. - - **Raises**: - - `KeyError`: If input keys are not found in the state, indicating that the necessary information for generating tag predictions is missing. - -#### Example Usage - -Here is an example of how to use the `GetProbableTagsNode` class: - -```python -from get_probable_tags_node import GetProbableTagsNode - -# Define a GetProbableTagsNode -get_probable_tags_node = GetProbableTagsNode( - input="user_input & document", - output=["probable_tags"], - node_config={"llm_model": llm_model}, - node_name="GetProbableTags" -) - -# Define the state -state = { - "user_input": "What tags contain relevant information for the query?", - "document": [document], - "url": "https://example.com" -} - -# Execute the GetProbableTagsNode -state = get_probable_tags_node.execute(state) - -# Retrieve the probable tags from the state -probable_tags = state["probable_tags"] - -print(f"Probable Tags: {probable_tags}") diff --git a/docs/Nodes/html_analyzer_node.md b/docs/Nodes/html_analyzer_node.md deleted file mode 100644 index e4f4ecf5..00000000 --- a/docs/Nodes/html_analyzer_node.md +++ /dev/null @@ -1,68 +0,0 @@ -# ๐ŸŒ HtmlAnalyzerNode Module - -The `HtmlAnalyzerNode` module analyzes HTML code based on the desired information to be extracted and returns a reduced version of the HTML content. The output of this node is primarily intended to enhance the reasoning capabilities of the `GenerateCodeNode`, enabling it to generate more accurate and context-aware code. - -## Classes - -### `HtmlAnalyzerNode` - -`HtmlAnalyzerNode` generates an analysis of the HTML code and returns a reduced version of the HTML, primarily to improve the `GenerateCodeNode`'s reasoning. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **additional_info**: Additional context to be included in the HTML analysis. -- **reduction (int)**: The level of reduction to apply to the HTML content. - - 0: Minification only. - - 1: Minification and removal of unnecessary tags and attributes - - 2: Minification, removal of unnecessary tags and attributes, simplification of text content, and removal of the `` tag. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "HtmlAnalyzer")`** - - Initializes the `HtmlAnalyzerNode` with necessary configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "HtmlAnalyzer". - -- **`execute(self, state: dict) -> dict`** - - Generates an analysis of the provided HTML code based on the wanted information to be extracted and returns a reduced version of the HTML content. The analysis is primarily used to enhance the reasoning of the `GenerateCodeNode`. - - The information to be extracted can be either the user's original input or, preferably, the refined prompt from the `PromptRefinerNode` for better results. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the following output keys: - - The generated HTML analysis. - - The reduced HTML code based on the specified reduction strength. - - **Raises**: - - `KeyError`: If the input keys are not found in the state, indicating that the necessary information for generating an answer is missing. - -#### Example Usage - -Here is an example of how to use the `HtmlAnalyzerNode` class: - -```python -from html_analyzer_node import HtmlAnalyzerNode - -# Define a html analyzer node -html_analyzer_node = HtmlAnalyzerNode( - input="refined_prompt & original_html", - output=["html_info", "reduced_html"] -) - -# Define the state -state = {"refined_prompt": "What are the attractions in Chioggia?", - "original_html": "..."} - -# Execute the generate answer node -state = html_analyzer_node.execute(state) - -# Retrieve the generated answer from the state -html_info = state["html_info"] -reduced_html = state["reduced_html"] - -print(f"HTML Analysis: {html_info}") -``` \ No newline at end of file diff --git a/docs/Nodes/image_to_text_node.md b/docs/Nodes/image_to_text_node.md deleted file mode 100644 index 18c987b0..00000000 --- a/docs/Nodes/image_to_text_node.md +++ /dev/null @@ -1,59 +0,0 @@ -# ๐Ÿ˜† ImageToTextNode Module - -The `ImageToTextNode` module implements a node responsible for retrieving images from a list of URLs and returning a description of the images using an image-to-text model. - -## Classes - -### `ImageToTextNode` - -`ImageToTextNode` is a node responsible for retrieving images from a list of URLs and returning a description of the images using an image-to-text model. - -#### Attributes - -- **llm_model**: An instance of the language model client used for image-to-text conversion. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: dict, node_name: str = "ImageToText")`** - - Initializes the `ImageToTextNode` with a language model client and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "ImageToText". - -- **`execute(self, state: dict) -> dict`** - - Generate text from an image using an image-to-text model. The method retrieves the image from the list of URLs provided in the state and returns the extracted text. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data types from the state. - - **Returns**: - - `dict`: The updated state with the input key containing the text extracted from the image. - -#### Example Usage - -Here is an example of how to use the `ImageToTextNode` class: - -```python -from image_to_text_node import ImageToTextNode - -# Define an ImageToTextNode -image_to_text_node = ImageToTextNode( - input="image_urls", - output=["image_descriptions"], - node_config={"llm_model": llm_model}, - node_name="ImageToText" -) - -# Define the state -state = { - "image_urls": ["https://example.com/image1.jpg", "https://example.com/image2.jpg"], -} - -# Execute the ImageToTextNode -state = image_to_text_node.execute(state) - -# Retrieve the image descriptions from the state -image_descriptions = state["image_descriptions"] - -print(f"Image Descriptions: {image_descriptions}") diff --git a/docs/Nodes/merge_answer_node.md b/docs/Nodes/merge_answer_node.md deleted file mode 100644 index 1dad31dd..00000000 --- a/docs/Nodes/merge_answer_node.md +++ /dev/null @@ -1,60 +0,0 @@ -# ๐ŸฆŒMergeAnswersNode Module - -The `MergeAnswersNode` module implements a node responsible for merging the answers from multiple graph instances into a single answer. - -## Classes - -### `MergeAnswersNode` - -`MergeAnswersNode` is a node responsible for merging the answers from multiple graph instances into a single answer. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "MergeAnswers")`** - - Initializes the `MergeAnswersNode` with a language model client and a node name. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "MergeAnswers". - -- **`execute(self, state: dict) -> dict`** - - Executes the node's logic to merge the answers from multiple graph instances into a single answer. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the generated answer. - -#### Example Usage - -Here is an example of how to use the `MergeAnswersNode` class: - -```python -from merge_answers_node import MergeAnswersNode - -# Define a MergeAnswersNode -merge_answers_node = MergeAnswersNode( - input="user_prompt, website_contents", - output=["merged_answer"], - node_config={"llm_model": llm_model}, - node_name="MergeAnswers" -) - -# Define the state -state = { - "user_prompt": "What are the advantages of renewable energy?", - "website_contents": ["Advantages of renewable energy: ...", "Renewable energy is beneficial because ..."], -} - -# Execute the MergeAnswersNode -state = merge_answers_node.execute(state) - -# Retrieve the merged answer from the state -merged_answer = state["merged_answer"] - -print(f"Merged Answer: {merged_answer}") diff --git a/docs/Nodes/parse_text_node.md b/docs/Nodes/parse_text_node.md deleted file mode 100644 index b75a68b9..00000000 --- a/docs/Nodes/parse_text_node.md +++ /dev/null @@ -1,58 +0,0 @@ -# ๐Ÿ ParseNode Module - -The `ParseNode` module implements a node responsible for parsing HTML content from a document and splitting it into chunks for further processing. - -## Classes - -### `ParseNode` - -`ParseNode` is a node responsible for parsing HTML content from a document and splitting it into chunks for further processing. - -#### Attributes - -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "Parse")`** - - Initializes the `ParseNode` with a node name and other optional configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "Parse". - -- **`execute(self, state: dict) -> dict`** - - Executes the node's logic to parse the HTML document content and split it into chunks. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the parsed content chunks. - -#### Example Usage - -Here is an example of how to use the `ParseNode` class: - -```python -from parse_node import ParseNode - -# Define a ParseNode -parse_node = ParseNode( - input="document", - output=["chunks"], - node_config={"verbose": True, "parse_html": True}, - node_name="Parse" -) - -# Define the state -state = { - "document": ["

Hello, world!

This is a test.

"], -} - -# Execute the ParseNode -state = parse_node.execute(state) - -# Retrieve the parsed content chunks from the state -chunks = state["chunks"] - -print(f"Parsed Chunks: {chunks}") diff --git a/docs/Nodes/prompt_refiner_node.md b/docs/Nodes/prompt_refiner_node.md deleted file mode 100644 index ee6085e3..00000000 --- a/docs/Nodes/prompt_refiner_node.md +++ /dev/null @@ -1,60 +0,0 @@ -# ๐Ÿ” PromptRefinerNode Module - -The `PromptRefinerNode` module refines the user's initial prompt using the output schema and any additional provided context. It aims to create a more precise prompt that establishes explicit links between elements in the user's original input and their corresponding representations in the JSON schema. - -## Classes - -### `PromptRefinerNode` - -`PromptRefinerNode` refines the user's prompt using the output schema and any additional provided context. - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **additional_info**: Additional context to help the language model better understand the request. -- **output_schema**: The output schema to be used for prompt refinement. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "PromptRefiner")`** - - Initializes the `PromptRefinerNode` with necessary configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "PromptRefiner". - -- **`execute(self, state: dict) -> dict`** - - Generates a refined prompt using the user's prompt, the schema, and additional context. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the generated refined prompt. - - **Raises**: - - `KeyError`: If the input keys are not found in the state, indicating that the necessary information for generating an answer is missing. - -#### Example Usage - -Here is an example of how to use the `PromptRefinerNode` class: - -```python -from prompt_refiner_node import PromptRefinerNode - -# Define a prompt refiner node -prompt_refier_node = PromptRefinerNode( - input="user_prompt", - output=["refined_prompt"] -) - -# Define the state -state = {"user_input": "What are the attractions in Chioggia?"} - -# Execute the generate answer node -state = prompt_refier_node.execute(state) - -# Retrieve the generated answer from the state -refined_prompt = state["refined_prompt"] - -print(f"Refined Prompt: {refined_prompt}") -``` \ No newline at end of file diff --git a/docs/Nodes/rag.md b/docs/Nodes/rag.md deleted file mode 100644 index 8c87e017..00000000 --- a/docs/Nodes/rag.md +++ /dev/null @@ -1,60 +0,0 @@ -# ๐Ÿ– RAGNode Module - -The `RAGNode` module implements a node responsible for compressing input tokens and storing the document in a vector database for retrieval. Relevant chunks are stored in the state. - -## Classes - -### `RAGNode` - -`RAGNode` is a node responsible for implementing RAG (Retrieval-Augmented Generation). - -#### Attributes - -- **llm_model**: An instance of a language model client, configured for generating answers. -- **embedder_model**: An instance of an embedding model client, configured for generating embeddings. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "RAG")`** - - Initializes the `RAGNode` with a node name and other optional configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "RAG". - -- **`execute(self, state: dict) -> dict`** - - Executes the node's logic to implement RAG (Retrieval-Augmented Generation). The method updates the state with relevant chunks of the document. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the relevant chunks of the document. - -#### Example Usage - -Here is an example of how to use the `RAGNode` class: - -```python -from rag_node import RAGNode - -# Define a RAGNode -rag_node = RAGNode( - input="document", - output=["compressed_docs"], - node_config={"verbose": True}, - node_name="RAG" -) - -# Define the state -state = { - "document": ["

Hello, world!

This is a test.

"], -} - -# Execute the RAGNode -state = rag_node.execute(state) - -# Retrieve the relevant chunks of the document from the state -compressed_docs = state["compressed_docs"] - -print(f"Compressed Documents: {compressed_docs}") diff --git a/docs/Nodes/robots_node.md b/docs/Nodes/robots_node.md deleted file mode 100644 index 2f9efddc..00000000 --- a/docs/Nodes/robots_node.md +++ /dev/null @@ -1,60 +0,0 @@ -# ๐Ÿฆ‚ RobotsNode Module - -The `RobotsNode` module implements a node responsible for checking if a website is scrapeable or not based on the robots.txt file. It uses a language model to determine if the website allows scraping of the provided path. - -## Classes - -### `RobotsNode` - -`RobotsNode` is a node responsible for checking if a website is scrapeable or not based on the robots.txt file. - -#### Attributes - -- **llm_model**: An instance of the language model client used for checking scrapeability. -- **force_scraping (bool)**: A flag indicating whether scraping should be enforced even if disallowed by robots.txt. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "Robots")`** - - Initializes the `RobotsNode` with a node name and other optional configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "Robots". - -- **`execute(self, state: dict) -> dict`** - - Checks if a website is scrapeable based on the robots.txt file and updates the state with the scrapeability status. The method constructs a prompt for the language model, submits it, and parses the output to determine if scraping is allowed. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the scrapeability status. - -#### Example Usage - -Here is an example of how to use the `RobotsNode` class: - -```python -from robots_node import RobotsNode - -# Define a RobotsNode -robots_node = RobotsNode( - input="source", - output=["is_scrapable"], - node_config={"llm_model": ..., "force_scraping": False, "verbose": True}, - node_name="Robots" -) - -# Define the state -state = { - "source": "https://example.com", -} - -# Execute the RobotsNode -state = robots_node.execute(state) - -# Retrieve the scrapeability status from the state -is_scrapable = state["is_scrapable"] - -print(f"Is Scrapable: {is_scrapable}") diff --git a/docs/Nodes/search_internet_node.md b/docs/Nodes/search_internet_node.md deleted file mode 100644 index 04499902..00000000 --- a/docs/Nodes/search_internet_node.md +++ /dev/null @@ -1,64 +0,0 @@ -# ๐Ÿฆ‘ SearchInternetNode Module - -The `SearchInternetNode` module implements a node that generates a search query based on the user's input and searches the internet for relevant information. The node constructs a prompt for the language model, submits it, and processes the output to generate a search query. It then uses the search query to find relevant information on the internet and updates the state with the generated answer. - -## Classes - -### `SearchInternetNode` - -`SearchInternetNode` is a node that generates a search query based on the user's input and searches the internet for relevant information. - -#### Attributes - -- **llm_model**: An instance of the language model client used for generating search queries. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. -- **max_results (int)**: Maximum number of search results to retrieve. Defaults to 3. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "SearchInternet")`** - - Initializes the `SearchInternetNode` with a node name and other optional configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "SearchInternet". - -- **`execute(self, state: dict) -> dict`** - - Generates an answer by constructing a prompt from the user's input and the scraped content, querying the language model, and parsing its response. The method updates the state with the generated answer. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data types from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the generated answer. - - **Raises**: - - `ValueError`: If the input keys are not found in the state, indicating that the necessary information for generating the answer is missing. - -#### Example Usage - -Here is an example of how to use the `SearchInternetNode` class: - -```python -from search_internet_node import SearchInternetNode - -# Define a SearchInternetNode -search_internet_node = SearchInternetNode( - input="user_prompt", - output=["search_results"], - node_config={"llm_model": ..., "verbose": True, "max_results": 5}, - node_name="SearchInternet" -) - -# Define the state -state = { - "user_prompt": "How does climate change affect biodiversity?", -} - -# Execute the SearchInternetNode -state = search_internet_node.execute(state) - -# Retrieve the search results from the state -search_results = state["search_results"] - -print("Search Results:") -for result in search_results: - print(result) diff --git a/docs/Nodes/search_link_node.md b/docs/Nodes/search_link_node.md deleted file mode 100644 index 3a63d7af..00000000 --- a/docs/Nodes/search_link_node.md +++ /dev/null @@ -1,64 +0,0 @@ -# ๐Ÿฆ– SearchLinkNode Module - -The `SearchLinkNode` module implements a node that can filter out the relevant links in the webpage content for the user prompt. It expects already scraped links on the webpage, making it ideal to use after the `FetchNode`. - -## Classes - -### `SearchLinkNode` - -`SearchLinkNode` is a node responsible for filtering out the relevant links from the webpage content for the user prompt. - -#### Attributes - -- **llm_model**: An instance of the language model client used for generating answers. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "GenerateLinks")`** - - Initializes the `SearchLinkNode` with a node name and other optional configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "GenerateLinks". - -- **`execute(self, state: dict) -> dict`** - - Filter out relevant links from the webpage that are relevant to the prompt. Out of the filtered links, also ensure that all links are navigable. The method updates the state with the list of relevant links. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data types from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the list of links. - - **Raises**: - - `KeyError`: If the input keys are not found in the state, indicating that the necessary information for generating the answer is missing. - -#### Example Usage - -Here is an example of how to use the `SearchLinkNode` class: - -```python -from search_link_node import SearchLinkNode - -# Define a SearchLinkNode -search_link_node = SearchLinkNode( - input=["user_prompt", "parsed_content_chunks"], - output=["relevant_links"], - node_config={"llm_model": ..., "verbose": True}, - node_name="GenerateLinks" -) - -# Define the state -state = { - "user_prompt": "How does climate change affect biodiversity?", - "parsed_content_chunks": [chunk1, chunk2, ...], -} - -# Execute the SearchLinkNode -state = search_link_node.execute(state) - -# Retrieve the relevant links from the state -relevant_links = state["relevant_links"] - -print("Relevant Links:") -for link in relevant_links: - print(link) diff --git a/docs/Nodes/text_to_speach_node.md b/docs/Nodes/text_to_speach_node.md deleted file mode 100644 index 4db6a5bd..00000000 --- a/docs/Nodes/text_to_speach_node.md +++ /dev/null @@ -1,59 +0,0 @@ -# ๐Ÿฆ TextToSpeechNode Module - -The `TextToSpeechNode` module converts text to speech using the specified text-to-speech model. - -## Classes - -### `TextToSpeechNode` - -`TextToSpeechNode` is a node responsible for converting text to speech using the specified text-to-speech model. - -#### Attributes - -- **tts_model**: An instance of the text-to-speech model client. -- **verbose (bool)**: A flag indicating whether to show print statements during execution. - -#### Methods - -- **`__init__(self, input: str, output: List[str], node_config: Optional[dict] = None, node_name: str = "TextToSpeech")`** - - Initializes the `TextToSpeechNode` with a node name and other optional configurations. - - **Args**: - - `input (str)`: Boolean expression defining the input keys needed from the state. - - `output (List[str])`: List of output keys to be updated in the state. - - `node_config (dict, optional)`: Additional configuration for the node. Defaults to None. - - `node_name (str, optional)`: The unique identifier name for the node. Defaults to "TextToSpeech". - -- **`execute(self, state: dict) -> dict`** - - Converts text to speech using the specified text-to-speech model. - - **Args**: - - `state (dict)`: The current state of the graph. The input keys will be used to fetch the correct data types from the state. - - **Returns**: - - `dict`: The updated state with the output key containing the audio generated from the text. - - **Raises**: - - `KeyError`: If the input keys are not found in the state, indicating that the necessary information for generating the audio is missing. - -#### Example Usage - -Here is an example of how to use the `TextToSpeechNode` class: - -```python -from text_to_speech_node import TextToSpeechNode - -# Define a TextToSpeechNode -text_to_speech_node = TextToSpeechNode( - input=["text_to_translate"], - output=["audio"], - node_config={"tts_model": ..., "verbose": True}, - node_name="TextToSpeech" -) - -# Define the state -state = { - "text_to_translate": "Hello, how are you?", -} - -# Execute the TextToSpeechNode -state = text_to_speech_node.execute(state) - -# Retrieve the generated audio from the state -audio = state["audio"] diff --git a/docs/Info/_category_.json b/docs/Some numbers/_category_.json similarity index 100% rename from docs/Info/_category_.json rename to docs/Some numbers/_category_.json diff --git a/docs/Info/api.md b/docs/Some numbers/api.md similarity index 100% rename from docs/Info/api.md rename to docs/Some numbers/api.md diff --git a/docs/Info/stars.md b/docs/Some numbers/stars.md similarity index 100% rename from docs/Info/stars.md rename to docs/Some numbers/stars.md diff --git a/docs/Utils/_category_.json b/docs/Utils/_category_.json deleted file mode 100644 index 0457db48..00000000 --- a/docs/Utils/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Utils", - "position": 9, - "link": { - "type": "generated-index", - "description": "Description of the utils folder" - } -} \ No newline at end of file diff --git a/docs/Utils/convert_to_csv.md b/docs/Utils/convert_to_csv.md deleted file mode 100644 index 00f3bccb..00000000 --- a/docs/Utils/convert_to_csv.md +++ /dev/null @@ -1,102 +0,0 @@ -# ๐Ÿท convert_to_csv - -## Introduction -The convert_to_csv function takes a dictionary containing data, a filename, and a directory position, and saves the data in CSV format at the specified location. - -## Implementation -```python -""" -Module that given a filename and a position saves the file in the csv format -""" -import os -import sys -import pandas as pd - - -def convert_to_csv(data: dict, filename: str, position: str = None): - """ - Converts a dictionary to a CSV file and saves it. - - Args: - data (dict): Data to be converted to CSV. - position (str): Optional path where the file should be saved. If not provided, - the directory of the caller script will be used. - - Raises: - FileNotFoundError: If the specified directory does not exist. - PermissionError: If the program lacks write permission for the directory. - TypeError: If the input data is not a dictionary. - Exception: For other potential errors during DataFrame creation or CSV saving. - """ - - if ".csv" in filename: - filename = filename.replace(".csv", "") # Remove .csv extension - - # Get the directory of the caller script if position is not provided - if position is None: - caller_dir = os.path.dirname(os.path.abspath(sys.argv[0])) - position = caller_dir - - try: - if not isinstance(data, dict): - raise TypeError("Input data must be a dictionary") - - os.makedirs(position, exist_ok=True) # Create directory if needed - - df = pd.DataFrame.from_dict(data, orient='index') - df.to_csv(os.path.join(position, f"{filename}.csv"), index=False) - - except FileNotFoundError as fnfe: - raise FileNotFoundError( - f"The specified directory '{position}' does not exist.") from fnfe - except PermissionError as pe: - raise PermissionError( - f"You don't have permission to write to '{position}'.") from pe - except Exception as e: - raise e # Re-raise other potential errors - -``` - -## Example -```python -""" -Teest for convert_to_csv -""" -import os -from scrapegraphai.utils.convert_to_csv import convert_to_csv - - -def main(): - """ - Example usage of the convert_to_csv function. - """ - # Example data - data = { - 'Name': ['John', 'Alice', 'Bob'], - 'Age': [30, 25, 35], - 'City': ['New York', 'San Francisco', 'Seattle'] - } - - # Example filename and position - filename = "example_data" - position = "./output" - - try: - # Convert data to CSV and save - convert_to_csv(data, filename, position) - print( - f"Data saved successfully to {os.path.join(position, filename)}.csv") - except ValueError as ve: - print(f"ValueError: {ve}") - except FileNotFoundError as fnfe: - print(f"FileNotFoundError: {fnfe}") - except PermissionError as pe: - print(f"PermissionError: {pe}") - except Exception as e: - print(f"An unexpected error occurred: {e}") - - -if __name__ == "__main__": - main() - -``` \ No newline at end of file diff --git a/docs/Utils/convert_to_j.md b/docs/Utils/convert_to_j.md deleted file mode 100644 index f38b9be6..00000000 --- a/docs/Utils/convert_to_j.md +++ /dev/null @@ -1,82 +0,0 @@ -# ๐Ÿค– convert_to_json -## Introduction -The convert_to_json function takes a dictionary containing data, a filename, and a directory position, and saves the data in JSON format at the specified location. -## Implementation -```python -""" -Convert to json module -""" -import json -import os -import sys - - -def convert_to_json(data: dict, filename: str, position: str = None): - """ - Convert data to JSON format and save it to a file. - - Args: - data (dict): Data to save. - filename (str): Name of the file to save without .json extension. - position (str): Directory where the file should be saved. If None, - the directory of the caller script will be used. - - Raises: - ValueError: If filename contains '.json'. - FileNotFoundError: If the specified directory does not exist. - PermissionError: If the program does not have permission to write to the directory. - """ - if ".json" in filename: - filename = filename.replace(".json", "") # Remove .csv extension - - # Get the directory of the caller script - if position is None: - # Get directory of the main script - caller_dir = os.path.dirname(os.path.abspath(sys.argv[0])) - position = caller_dir - - try: - os.makedirs(position, exist_ok=True) - with open(os.path.join(position, f"{filename}.json"), "w", encoding="utf-8") as f: - f.write(json.dumps(data)) - except FileNotFoundError as fnfe: - raise FileNotFoundError( - f"The specified directory '{position}' does not exist.") from fnfe - except PermissionError as pe: - raise PermissionError( - f"You don't have permission to write to '{position}'.") from pe - -``` - -## Example -```python -""" -Example of using convert_to_json function to save data in JSON format. -""" -import os -from scrapegraphai.utils.convert_to_json import convert_to_json - -# Data to save in JSON format -data_to_save = { - "name": "John Doe", - "age": 30, - "city": "New York" -} - -FILENAME = "example_data" -DIRECTORY = "data_output" - -try: - convert_to_json(data_to_save, FILENAME, DIRECTORY) - print( - f"Data has been successfully saved to {os.path.join(DIRECTORY, FILENAME)}.json") -except ValueError as value_error: - print(value_error) -except FileNotFoundError as file_not_found_error: - print(file_not_found_error) -except PermissionError as permission_error: - print(permission_error) -except Exception as exception: - print(f"An error occurred: {exception}") - -``` \ No newline at end of file diff --git a/docs/Utils/parse_state_key.md b/docs/Utils/parse_state_key.md deleted file mode 100644 index cff8921c..00000000 --- a/docs/Utils/parse_state_key.md +++ /dev/null @@ -1,106 +0,0 @@ -# ๐Ÿ“ก Parse state -Function for parsing the state of the key - -## Implementation -```python -""" -Parse_state_key module -""" -import re - - -def parse_expression(expression, state: dict): - """ - Function for parsing the expressions - Args: - state (dict): state to elaborate - """ - # Check for empty expression - if not expression: - raise ValueError("Empty expression.") - - # Check for adjacent state keys without an operator between them - pattern = r'\b(' + '|'.join(re.escape(key) for key in state.keys()) + \ - r')(\b\s*\b)(' + '|'.join(re.escape(key) - for key in state.keys()) + r')\b' - if re.search(pattern, expression): - raise ValueError( - "Adjacent state keys found without an operator between them.") - - # Remove spaces - expression = expression.replace(" ", "") - - # Check for operators with empty adjacent tokens or at the start/end - if expression[0] in '&|' or expression[-1] in '&|' or \ - '&&' in expression or '||' in expression or \ - '&|' in expression or '|&' in expression: - - raise ValueError("Invalid operator usage.") - - # Check for balanced parentheses and valid operator placement - open_parentheses = close_parentheses = 0 - for i, char in enumerate(expression): - if char == '(': - open_parentheses += 1 - elif char == ')': - close_parentheses += 1 - # Check for invalid operator sequences - if char in "&|" and i + 1 < len(expression) and expression[i + 1] in "&|": - raise ValueError( - "Invalid operator placement: operators cannot be adjacent.") - - # Check for missing or balanced parentheses - if open_parentheses != close_parentheses: - raise ValueError("Missing or unbalanced parentheses in expression.") - - # Helper function to evaluate an expression without parentheses - def evaluate_simple_expression(exp): - # Split the expression by the OR operator and process each segment - for or_segment in exp.split('|'): - # Check if all elements in an AND segment are in state - and_segment = or_segment.split('&') - if all(elem.strip() in state for elem in and_segment): - return [elem.strip() for elem in and_segment if elem.strip() in state] - return [] - - # Helper function to evaluate expressions with parentheses - def evaluate_expression(expression): - while '(' in expression: - start = expression.rfind('(') - end = expression.find(')', start) - sub_exp = expression[start + 1:end] - # Replace the evaluated part with a placeholder and then evaluate it - sub_result = evaluate_simple_expression(sub_exp) - # For simplicity in handling, join sub-results with OR to reprocess them later - expression = expression[:start] + \ - '|'.join(sub_result) + expression[end+1:] - return evaluate_simple_expression(expression) - - temp_result = evaluate_expression(expression) - - if not temp_result: - raise ValueError("No state keys matched the expression.") - - # Remove redundant state keys from the result, without changing their order - final_result = [] - for key in temp_result: - if key not in final_result: - final_result.append(key) - - return final_result - - -EXPRESSION = "user_input & (relevant_chunks | parsed_document | document)" -state = { - "user_input": None, - "document": None, - "parsed_document": None, - "relevant_chunks": None, -} - -try: - result = parse_expression(EXPRESSION, state) - print("Matched keys:", result) -except ValueError as e: - print("Error:", e) -``` \ No newline at end of file diff --git a/docs/Utils/prettify_info.md b/docs/Utils/prettify_info.md deleted file mode 100644 index 2715a301..00000000 --- a/docs/Utils/prettify_info.md +++ /dev/null @@ -1,94 +0,0 @@ -# ๐ŸŒง๏ธ prettify_exec_info - -## Introduction -Function that gives info about cost and time about the scraper call - -## Implementation - -```python -""" -Prettify the execution information of the graph. -""" - -import pandas as pd - - -def prettify_exec_info(complete_result: dict) -> pd.DataFrame: - """ - Transform the execution information of the graph into a DataFrame for better visualization. - - Args: - - complete_result (dict): The complete execution information of the graph. - - Returns: - - pd.DataFrame: The execution information of the graph in a DataFrame. - """ - - nodes_info = complete_result['nodes_info'] - total_info = { - 'total_exec_time': complete_result['total_exec_time'], - 'total_model_info': complete_result['total_model_info'] - } - - # Convert node-specific information to DataFrame - flat_data = [] - for node_name, node_info in nodes_info.items(): - flat_data.append({ - 'Node': node_name, - 'Execution Time': node_info['exec_time'], - # Unpack the model_info dict into the row - **node_info['model_info'] - }) - - df_nodes = pd.DataFrame(flat_data) - - # Add a row for the total execution time and total model info - total_row = { - 'Node': 'Total', - 'Execution Time': total_info['total_exec_time'], - # Unpack the total_model_info dict into the row - **total_info['total_model_info'] - } - df_total = pd.DataFrame([total_row]) - - # Combine the nodes DataFrame with the total info DataFrame - df_combined_with_total = pd.concat([df_nodes, df_total], ignore_index=True) - return df_combined_with_total -``` - -## Example - -```python - -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 1, - "format": "json", # Ollama needs the format to be specified explicitly - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - } -} - - - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the news with their description.", - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -graph_exec_info = smart_scraper_graph.get_execution_info() -print(prettify_exec_info(graph_exec_info)) - -``` diff --git a/docs/Utils/r.md b/docs/Utils/r.md deleted file mode 100644 index 350f0e3a..00000000 --- a/docs/Utils/r.md +++ /dev/null @@ -1,71 +0,0 @@ -# ๐Ÿšฃ Remover - -## Introduction -The remover is a module that given an html string it removes all the tags that are not necessary - -## Implementation -```python -""" -Module for minimizing the code -""" -from bs4 import BeautifulSoup -from minify_html import minify - - -def remover(html_content: str) -> str: - """ - This function processes HTML content, removes unnecessary tags - (including style tags), minifies the HTML, and retrieves the - title and body content. - - Parameters: - html_content (str): The HTML content to parse - - Returns: - str: The parsed title followed by the minified body content - """ - - soup = BeautifulSoup(html_content, 'html.parser') - - # Title Extraction - title_tag = soup.find('title') - title = title_tag.get_text() if title_tag else "" - - # Script and Style Tag Removal - for tag in soup.find_all(['script', 'style']): - tag.extract() - - # Body Extraction (if it exists) - body_content = soup.find('body') - if body_content: - # Minify the HTML within the body tag - minimized_body = minify(str(body_content)) - return "Title: " + title + ", Body: " + minimized_body - - return "Title: " + title + ", Body: No body content found" -``` -## Example -```python -""" -Example of the remover method -""" -from scrapegraphai.utils.remover import remover - -HTML_CONTENT = """ - - - Test Page - - -

This is a Test

-

Hello, World!

- - - -""" - -parsed_content = remover(HTML_CONTENT) - -print(parsed_content) - -``` \ No newline at end of file diff --git a/docs/Utils/save.md b/docs/Utils/save.md deleted file mode 100644 index 30db8efd..00000000 --- a/docs/Utils/save.md +++ /dev/null @@ -1,43 +0,0 @@ -# ๐Ÿ† save_audio_from_bytes - -## Introduction -This function given a filename, and a path it saves the audio file - -## Implementation -```python -""" -This utility function saves the byte response as an audio file. -""" - -from pathlib import Path - - -def save_audio_from_bytes(byte_response, output_path): - """ - Saves the byte response as an audio file. - - Args: - byte_response (bytes): The byte response containing the generated speech. - output_path (str or Path): The file path where the generated speech should be saved. - """ - if not isinstance(output_path, Path): - output_path = Path(output_path) - - with open(output_path, 'wb') as audio_file: - audio_file.write(byte_response) -``` - -## Example -```python -""" -Example for th e file save_audio_from_bytes -""" -from scrapegraphai.utils.save_audio_from_bytes import save_audio_from_bytes - -BYTE_RESPONSE = b'\x12\x34\x56\x78\x90' - -OUTPUT_PATH = "generated_speech.wav" - -save_audio_from_bytes(BYTE_RESPONSE, OUTPUT_PATH) - -``` \ No newline at end of file diff --git a/docs/Utils/saveaudio.md b/docs/Utils/saveaudio.md deleted file mode 100644 index 2fe57f8a..00000000 --- a/docs/Utils/saveaudio.md +++ /dev/null @@ -1,26 +0,0 @@ -# ๐Ÿ”ˆ Save audio -Function for saving audio - -## Implementation -```python -""" -This utility function saves the byte response as an audio file. -""" -from pathlib import Path -from typing import Union - - -def save_audio_from_bytes(byte_response: bytes, output_path: Union[str, Path]) -> None: - """ - Saves the byte response as an audio file. - - Args: - byte_response (bytes): The byte response containing the generated speech. - output_path (str or Path): The file path where the generated speech should be saved. - """ - if not isinstance(output_path, Path): - output_path = Path(output_path) - - with open(output_path, 'wb') as audio_file: - audio_file.write(byte_response) -``` \ No newline at end of file diff --git a/docs/Utils/search_function.md b/docs/Utils/search_function.md deleted file mode 100644 index 8b03348a..00000000 --- a/docs/Utils/search_function.md +++ /dev/null @@ -1,52 +0,0 @@ -# ๐Ÿชฎ Search function -## Introduction -Function for searching in the internet - -## Implementation - -```python -""" -Module for making the request on the web -""" -import re -from typing import List -from langchain_community.tools import DuckDuckGoSearchResults -from googlesearch import search - - -def search_on_web(query: str, search_engine: str = "Google", max_results: int = 10) -> List[str]: - """ - Function that given a query it finds it on the intenet - Args: - query (str): query to search on internet - search_engine (str, optional): type of browser, it could be DuckDuckGo or Google, - default: Google - max_results (int, optional): maximum number of results - - Returns: - List[str]: List of strings of web link - """ - - if search_engine == "Google": - res = [] - - for url in search(query, stop=max_results): - res.append(url) - return res - elif search_engine == "DuckDuckGo": - research = DuckDuckGoSearchResults(max_results=max_results) - res = research.run(query) - - links = re.findall(r'https?://[^\s,\]]+', res) - - return links - raise ValueError( - "The only search engines avaiable are DuckDuckGo or Google") -``` - -## Example of usage -```python -from scrapegraphai.utils import search_on_web - -res = search_on_web("regions of Italy") -``` \ No newline at end of file diff --git a/docs/Utils/token_calculator.md b/docs/Utils/token_calculator.md deleted file mode 100644 index 686c8b23..00000000 --- a/docs/Utils/token_calculator.md +++ /dev/null @@ -1,55 +0,0 @@ -# ๐Ÿ‡น๐Ÿ‡ฐ Token calculator -The token calculator class is a class that implements the token for the link to scrape. -Given a link it retrieves a a list of chunks given the name of the model. - -## Implementation -```python -""" -Module for truncatinh in chunks the messages -""" -from typing import List -import tiktoken -from ..helpers.models_tokens import models_tokens - - -def truncate_text_tokens(text: str, model: str, encoding_name: str) -> List[str]: - """ - It creates a list of strings to create max dimension tokenizable elements - - Args: - text (str): The input text to be truncated into tokenizable elements. - model (str): The name of the language model to be used. - encoding_name (str): The name of the encoding to be used (default: EMBEDDING_ENCODING). - - Returns: - List[str]: A list of tokenizable elements created from the input text. - """ - - encoding = tiktoken.get_encoding(encoding_name) - max_tokens = models_tokens[model] - 500 - encoded_text = encoding.encode(text) - - chunks = [encoded_text[i:i + max_tokens] - for i in range(0, len(encoded_text), max_tokens)] - - result = [encoding.decode(chunk) for chunk in chunks] - - return result -``` -## Example -```python -""" -Example for calclating the tokenizer -""" -from scrapegraphai.utils.token_calculator import truncate_text_tokens -INPUT_TEXT = "http://nba.com" - -MODEL_NAME = "gpt-3.5-turbo" -ENCODING_NAME = "EMBEDDING_ENCODING" - -tokenized_chunks = truncate_text_tokens(INPUT_TEXT, MODEL_NAME, ENCODING_NAME) - -for i, chunk in enumerate(tokenized_chunks): - print(f"Chunk {i+1}: {chunk}") -``` - diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..2f81f45c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,41 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +# -- Path setup -------------------------------------------------------------- + +import os +import sys + +# import all the modules +sys.path.insert(0, os.path.abspath('../../')) + +project = 'ScrapeGraphAI' +copyright = '2024, ScrapeGraphAI' +author = 'Marco Vinciguerra, Marco Perini, Lorenzo Padoan' + +html_last_updated_fmt = "%b %d, %Y" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] + +templates_path = ['_templates'] +exclude_patterns = [] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'furo' +html_theme_options = { + "source_repository": "https://github.com/VinciGit00/Scrapegraph-ai/", + "source_branch": "main", + "source_directory": "docs/source/", + 'navigation_with_keys': True, + 'sidebar_hide_name': False, +} \ No newline at end of file diff --git a/docs/contributing/contributing_markdown.md b/docs/contributing/contributing_markdown.md deleted file mode 100644 index 45010c07..00000000 --- a/docs/contributing/contributing_markdown.md +++ /dev/null @@ -1,3 +0,0 @@ -# ๐ŸŽป List of the contributors -[![Contributors](https://contrib.rocks/image?repo=VinciGit00/Scrapegraph-ai)](https://github.com/VinciGit00/Scrapegraph-ai/graphs/contributors) - diff --git a/docs/core/_category_.json b/docs/core/_category_.json deleted file mode 100644 index dd5c6570..00000000 --- a/docs/core/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Core Components", - "position": 2, - "link": { - "type": "generated-index", - "description": "Documentation for the core components of Scrapegraph-ai, including Builders, Graphs, and Nodes." - } -} diff --git a/docs/getting_started/_category_.json b/docs/getting_started/_category_.json new file mode 100644 index 00000000..b3c262c6 --- /dev/null +++ b/docs/getting_started/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Getting Started", + "position": 2, + "link": { + "type": "generated-index", + "description": "Get started with ScrapeGraphAI by following these guides." + } +} \ No newline at end of file diff --git a/docs/getting_started/examples.md b/docs/getting_started/examples.md new file mode 100644 index 00000000..071bb857 --- /dev/null +++ b/docs/getting_started/examples.md @@ -0,0 +1,87 @@ +--- +sidebar_position: 3 +--- + +# Examples + +Let's suppose you want to scrape a website to get a list of projects with their descriptions. +You can use the `SmartScraperGraph` class to do that. +The following examples show how to use the `SmartScraperGraph` class with OpenAI models and local models. + +## OpenAI models + +```python +import os +from dotenv import load_dotenv +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info + +load_dotenv() + +openai_key = os.getenv("OPENAI_APIKEY") + +graph_config = { + "llm": { + "api_key": openai_key, + "model": "openai/gpt-4o", + }, +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me all the projects with their description.", + # also accepts a string with the already downloaded HTML code + source="https://perinim.github.io/projects/", + config=graph_config +) + +result = smart_scraper_graph.run() +print(result) +``` + +## Local models + +Remember to have installed in your pc [ollama](https://ollama.com/) +Remember to pull the right model for LLM and for the embeddings, like: + +```bash +ollama pull llama3 +ollama pull nomic-embed-text +ollama pull mistral +``` + +After that, you can run the following code, using only your machine resources brum brum brum: + +```python +from scrapegraphai.graphs import SmartScraperGraph +from scrapegraphai.utils import prettify_exec_info + +graph_config = { + "llm": { + "model": "ollama/mistral", + "temperature": 1, + "format": "json", # Ollama needs the format to be specified explicitly + "model_tokens": 2000, # depending on the model set context length + "base_url": "http://localhost:11434", # set ollama URL of the local host (YOU CAN CHANGE IT, if you have a different endpoint + } +} + +# ************************************************ +# Create the SmartScraperGraph instance and run it +# ************************************************ + +smart_scraper_graph = SmartScraperGraph( + prompt="List me all the projects with their description.", + # also accepts a string with the already downloaded HTML code + source="https://perinim.github.io/projects", + config=graph_config +) + +result = smart_scraper_graph.run() +print(result) +``` + +To find out how you can customize the `graph_config` dictionary, by using different LLM and adding new parameters, check the `Scrapers` section! \ No newline at end of file diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md new file mode 100644 index 00000000..7b59a5a7 --- /dev/null +++ b/docs/getting_started/installation.md @@ -0,0 +1,43 @@ +--- +sidebar_position: 2 +--- + +# Installation + +In the following sections I will guide you through the installation process of the required components +for this project. + +## Prerequisites + +- [Python >=3.9](https://www.python.org/downloads/) +- [pip](https://pip.pypa.io/en/stable/getting-started/) +- [Ollama](https://ollama.com/) (optional for local models) + +## Install the library + +The library is available on PyPI, so it can be installed using the following command: + +```bash +pip install scrapegraphai +``` + +> **Important**: It is highly recommended to install the library in a virtual environment (conda, venv, etc.) + +If you clone the repository, it is recommended to use a package manager like [uv](https://github.com/astral-sh/uv). +To install the library using uv, you can run the following command: + +```bash +uv pin 3.10 +uv sync +uv build +``` + +> **Caution**: **uv** must be installed first by following the instructions on the [official website](https://github.com/astral-sh/uv). + +## Additionally on Windows when using WSL + +If you are using Windows Subsystem for Linux (WSL) and you are facing issues with the installation of the library, you might need to install the following packages: + +```bash +sudo apt-get -y install libnss3 libnspr4 libgbm1 libasound2 +``` \ No newline at end of file diff --git a/docs/intro.md b/docs/intro.md index 67a9a2bc..78725ed0 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -1,280 +1,163 @@ +--- +sidebar_position: 1 +slug: / +--- + +# Overview + +# ScrapeGraphAI + +ScrapeGraphAI is an **open-source** Python library designed to revolutionize **scraping** tools. +In today's data-intensive digital landscape, this library stands out by integrating **Large Language Models** (LLMs) +and modular **graph-based** pipelines to automate the scraping of data from various sources (e.g., websites, local files etc.). + +Simply specify the information you need to extract, and ScrapeGraphAI handles the rest, providing a more **flexible** and **low-maintenance** solution compared to traditional scraping tools. + +## Why ScrapegraphAI? + +Traditional web scraping tools often rely on fixed patterns or manual configuration to extract data from web pages. +ScrapegraphAI, leveraging the power of LLMs, adapts to changes in website structures, reducing the need for constant developer intervention. +This flexibility ensures that scrapers remain functional even when website layouts change. + +We support many LLMs including **GPT, Gemini, Groq, Azure, Hugging Face** etc. +as well as local models which can run on your machine using **Ollama**. + +## AI Models and Token Limits + +ScrapGraphAI supports a wide range of AI models from various providers. Each model has a specific token limit, which is important to consider when designing your scraping pipelines. Here's an overview of the supported models and their token limits: + +### OpenAI Models +- GPT-3.5 Turbo (16,385 tokens) +- GPT-3.5 (4,096 tokens) +- GPT-3.5 Turbo Instruct (4,096 tokens) +- GPT-4 Turbo Preview (128,000 tokens) +- GPT-4 Vision Preview (128,000 tokens) +- GPT-4 (8,192 tokens) +- GPT-4 32k (32,768 tokens) +- GPT-4o (128,000 tokens) +- O1 Preview (128,000 tokens) +- O1 Mini (128,000 tokens) -# ๐Ÿ•ท๏ธ ScrapeGraphAI: You Only Scrape Once -[![Downloads](https://static.pepy.tech/badge/scrapegraphai)](https://pepy.tech/project/scrapegraphai) -[![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint) -[![Pylint](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml) -[![CodeQL](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![](https://dcbadge.vercel.app/api/server/gkxQDAjfeX)](https://discord.gg/gkxQDAjfeX) +### Azure OpenAI Models +- GPT-3.5 Turbo (16,385 tokens) +- GPT-3.5 (4,096 tokens) +- GPT-4 Turbo Preview (128,000 tokens) +- GPT-4 (8,192 tokens) +- GPT-4 32k (32,768 tokens) +- GPT-4o (128,000 tokens) +- O1 Preview (128,000 tokens) +- O1 Mini (128,000 tokens) + +### Google AI Models +- Gemini Pro (128,000 tokens) +- Gemini 1.5 Flash (128,000 tokens) +- Gemini 1.5 Pro (128,000 tokens) +- Gemini 1.0 Pro (128,000 tokens) +### Anthropic Models +- Claude Instant (100,000 tokens) +- Claude 2 (9,000 tokens) +- Claude 2.1 (200,000 tokens) +- Claude 3 (200,000 tokens) +- Claude 3.5 (200,000 tokens) +- Claude 3 Opus (200,000 tokens) +- Claude 3 Sonnet (200,000 tokens) +- Claude 3 Haiku (200,000 tokens) -ScrapeGraphAI is a *web scraping* python library that uses LLM and direct graph logic to create scraping pipelines for websites, documents and XML files. -Just say which information you want to extract and the library will do it for you! +### Mistral AI Models +- Mistral Large Latest (128,000 tokens) +- Open Mistral Nemo (128,000 tokens) +- Codestral Latest (32,000 tokens) +- Open Mistral 7B (32,000 tokens) +- Open Mixtral 8x7B (32,000 tokens) +- Open Mixtral 8x22B (64,000 tokens) +- Open Codestral Mamba (256,000 tokens) -[![My Skills](https://skillicons.dev/icons?i=discord)](https://discord.gg/gkxQDAjfeX) -[![My Skills](https://skillicons.dev/icons?i=linkedin)](https://www.linkedin.com/company/scrapegraphai/) -[![My Skills](https://skillicons.dev/icons?i=twitter)](https://twitter.com/scrapegraphai) +### Ollama Models +- Command-R (12,800 tokens) +- CodeLlama (16,000 tokens) +- DBRX (32,768 tokens) +- DeepSeek Coder 33B (16,000 tokens) +- Llama2 Series (4,096 tokens) +- Llama3 Series (8,192-128,000 tokens) +- Mistral Models (32,000-128,000 tokens) +- Mixtral 8x22B Instruct (65,536 tokens) +- Phi3 Series (12,800-128,000 tokens) +- Qwen Series (32,000 tokens) +### Hugging Face Models +- Grok-1 (8,192 tokens) +- Meta Llama 3 Series (8,192 tokens) +- Google Gemma Series (8,192 tokens) +- Microsoft Phi Series (2,048-131,072 tokens) +- GPT-2 Series (1,024 tokens) +- DeepSeek V2 Series (131,072 tokens) -## ๐Ÿš€ Quick install +### Bedrock Models +- Claude 3 Series (200,000 tokens) +- Llama2 & Llama3 Series (4,096-8,192 tokens) +- Mistral Series (32,768 tokens) +- Titan Embed Text (8,000 tokens) +- Cohere Embed (512 tokens) -The reference page for Scrapegraph-ai is available on the official page of pypy: [pypi](https://pypi.org/project/scrapegraphai/). +### Fireworks Models +- Llama V2 7B (4,096 tokens) +- Mixtral 8x7B Instruct (4,096 tokens) +- Llama 3.1 Series (131,072 tokens) +- Mixtral MoE Series (65,536 tokens) -```bash -pip install scrapegraphai -``` -you will also need to install Playwright for javascript-based scraping: -```bash -playwright install -``` +For a complete and up-to-date list of supported models and their token limits, please refer to the API documentation. -**Note**: it is recommended to install the library in a virtual environment to avoid conflicts with other libraries ๐Ÿฑ +Understanding token limits is crucial for optimizing your scraping tasks. Larger token limits allow for processing more text in a single API call, which can be beneficial for scraping lengthy web pages or documents. -## News ๐Ÿ“ฐ +## Library Diagram -- ScrapegraphAI has now his APIs! Check it out [here](https://scrapegraphai.com)! +With ScrapegraphAI you can use many already implemented scraping pipelines or create your own. + +The library follows a modular, graph-based architecture that enables flexible and efficient data extraction pipelines. + +## FAQ + +1. **What is ScrapeGraphAI?** + + ScrapeGraphAI is an open-source python library that uses large language models (LLMs) and graph logic to automate the creation of scraping pipelines for websites and various document types. + +2. **How does ScrapeGraphAI differ from traditional scraping tools?** + + Traditional scraping tools rely on fixed patterns and manual configurations, whereas ScrapeGraphAI adapts to website structure changes using LLMs, reducing the need for constant developer intervention. + +3. **Which LLMs are supported by ScrapeGraphAI?** + + ScrapeGraphAI supports several LLMs, including GPT, Gemini, Groq, Azure, Hugging Face, and local models that can run on your machine using Ollama. + +4. **Can ScrapeGraphAI handle different document formats?** + + Yes, ScrapeGraphAI can scrape information from various document formats such as XML, HTML, JSON, and more. + +5. **I get an empty or incorrect output when scraping a website. What should I do?** -## ๐Ÿ” Demo -Official streamlit demo: + There are several reasons behind this issue, but for most cases, you can try the following: -[![My Skills](https://skillicons.dev/icons?i=react)](https://scrapegraph-demo-demo.streamlit.app) + - Set the `headless` parameter to `False` in the graph_config. Some javascript-heavy websites might require it. + - Check your internet connection. Low speed or unstable connection can cause the HTML to not load properly. + - Try using a proxy server to mask your IP address. Check out the Proxy section for more information on how to configure proxy settings. + - Use a different LLM model. Some models might perform better on certain websites than others. + - Set the `verbose` parameter to `True` in the graph_config to see more detailed logs. + - Visualize the pipeline graphically using Burr. + + If the issue persists, please report it on the GitHub repository. -Try it directly on the web using Google Colab: +6. **How does ScrapeGraphAI handle the context window limit of LLMs?** -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1sEZBonBMGP44CtO6GQTwAlL0BGJXjtfd?usp=sharing) + By splitting big websites/documents into chunks with overlaps and applying compression techniques to reduce the number of tokens. If multiple chunks are present, we will have multiple answers to the user prompt, and therefore, we merge them together in the last step of the scraping pipeline. -Follow the procedure on the following link to setup your OpenAI API key: [link](https://scrapegraph-ai.readthedocs.io/en/latest/index.html). +7. **How can I contribute to ScrapeGraphAI?** -## ๐Ÿ“– Documentation + You can contribute to ScrapeGraphAI by submitting bug reports, feature requests, or pull requests on the GitHub repository. Join our [Discord](https://discord.gg/uJN7TYcpNa) community and follow us on social media! -The documentation for ScrapeGraphAI can be found [here](https://scrapegraph-ai.readthedocs.io/en/latest/). +## Sponsors -Check out also the docusaurus [documentation](https://scrapegraph-doc.onrender.com/). - -## ๐Ÿ’ป Usage -You can use the `SmartScraper` class to extract information from a website using a prompt. - -The `SmartScraper` class is a direct graph implementation that uses the most common nodes present in a web scraping pipeline. For more information, please see the [documentation](https://scrapegraph-ai.readthedocs.io/en/latest/). -### Case 1: Extracting information using Ollama -Remember to download the model on Ollama separately! - -```python -from scrapegraphai.graphs import SmartScraperGraph - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - "base_url": "http://localhost:11434", # set Ollama URL - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "base_url": "http://localhost:11434", # set Ollama URL - } -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the articles", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) - -``` - -### Case 2: Extracting information using Docker - -Note: before using the local model remember to create the docker container! -```text - docker-compose up -d - docker exec -it ollama ollama pull stablelm-zephyr -``` -You can use which models available on Ollama or your own model instead of stablelm-zephyr -```python -from scrapegraphai.graphs import SmartScraperGraph - -graph_config = { - "llm": { - "model": "ollama/mistral", - "temperature": 0, - "format": "json", # Ollama needs the format to be specified explicitly - # "model_tokens": 2000, # set context length arbitrarily - }, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the articles", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) -``` - - -### Case 3: Extracting information using Openai model -```python -from scrapegraphai.graphs import SmartScraperGraph -OPENAI_API_KEY = "YOUR_API_KEY" - -graph_config = { - "llm": { - "api_key": OPENAI_API_KEY, - "model": "gpt-3.5-turbo", - }, -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the articles", - # also accepts a string with the already downloaded HTML code - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) -``` - -### Case 4: Extracting information using Groq -```python -from scrapegraphai.graphs import SmartScraperGraph -from scrapegraphai.utils import prettify_exec_info - -groq_key = os.getenv("GROQ_APIKEY") - -graph_config = { - "llm": { - "model": "groq/gemma-7b-it", - "api_key": groq_key, - "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, - "headless": False -} - -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the projects with their description and the author.", - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) -``` - -### Case 5: Extracting information using Azure -```python -from langchain_openai import AzureChatOpenAI -from langchain_openai import AzureOpenAIEmbeddings - -lm_model_instance = AzureChatOpenAI( - openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], - azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] -) - -embedder_model_instance = AzureOpenAIEmbeddings( - azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"], - openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], -) -graph_config = { - "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} -} - -smart_scraper_graph = SmartScraperGraph( - prompt="""List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, - event_end_date, event_end_time, location, event_mode, event_category, - third_party_redirect, no_of_days, - time_in_hours, hosted_or_attending, refreshments_type, - registration_available, registration_link""", - source="https://www.hmhco.com/event", - config=graph_config -) -``` - -### Case 6: Extracting information using Gemini -```python -from scrapegraphai.graphs import SmartScraperGraph -GOOGLE_APIKEY = "YOUR_API_KEY" - -# Define the configuration for the graph -graph_config = { - "llm": { - "api_key": GOOGLE_APIKEY, - "model": "gemini-pro", - }, -} - -# Create the SmartScraperGraph instance -smart_scraper_graph = SmartScraperGraph( - prompt="List me all the articles", - source="https://perinim.github.io/projects", - config=graph_config -) - -result = smart_scraper_graph.run() -print(result) -``` - -The output for all 3 the cases will be a dictionary with the extracted information, for example: - -```bash -{ - 'titles': [ - 'Rotary Pendulum RL' - ], - 'descriptions': [ - 'Open Source project aimed at controlling a real life rotary pendulum using RL algorithms' - ] -} -``` - -## ๐Ÿค Contributing - -Feel free to contribute and join our Discord server to discuss with us improvements and give us suggestions! - -Please see the [contributing guidelines](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/CONTRIBUTING.md). - -## ๐Ÿ“ˆ Roadmap -Check out the project roadmap [here](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/docs/README.md)! ๐Ÿš€ - -Wanna visualize the roadmap in a more interactive way? Check out the [markmap](https://markmap.js.org/repl) visualization by copy pasting the markdown content in the editor! - -## โค๏ธ Contributors -[![Contributors](https://contrib.rocks/image?repo=VinciGit00/Scrapegraph-ai)](https://github.com/VinciGit00/Scrapegraph-ai/graphs/contributors) - -## ๐ŸŽ“ Citations -If you have used our library for research purposes please quote us with the following reference: -```text - @misc{scrapegraph-ai, - author = {Marco Perini, Lorenzo Padoan, Marco Vinciguerra}, - title = {Scrapegraph-ai}, - year = {2024}, - url = {https://github.com/VinciGit00/Scrapegraph-ai}, - note = {A Python library for scraping leveraging large language models} - } -``` -| | Contact Info | -|--------------------|----------------------| -| Marco Vinciguerra | [![Linkedin Badge](https://img.shields.io/badge/-Linkedin-blue?style=flat&logo=Linkedin&logoColor=white)](https://www.linkedin.com/in/marco-vinciguerra-7ba365242/) | -| Marco Perini | [![Linkedin Badge](https://img.shields.io/badge/-Linkedin-blue?style=flat&logo=Linkedin&logoColor=white)](https://www.linkedin.com/in/perinim/) | -| Lorenzo Padoan | [![Linkedin Badge](https://img.shields.io/badge/-Linkedin-blue?style=flat&logo=Linkedin&logoColor=white)](https://www.linkedin.com/in/lorenzo-padoan-4521a2154/) | - -## ๐Ÿ“œ License - -ScrapeGraphAI is licensed under the MIT License. See the [LICENSE](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/LICENSE) file for more information. - -## Acknowledgements - -- We would like to thank all the contributors to the project and the open-source community for their support. -- ScrapeGraphAI is meant to be used for data exploration and research purposes only. We are not responsible for any misuse of the library. +- [Browserbase](https://www.browserbase.com/) +- [Serp API](https://serpapi.com?utm_source=scrapegraphai) +- [Stat Proxies](https://dashboard.statproxies.com/?refferal=scrapegraph) +- [Scrapedo](https://scrape.do) diff --git a/docs/introduction/_category_.json b/docs/introduction/_category_.json new file mode 100644 index 00000000..c1963257 --- /dev/null +++ b/docs/introduction/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Introduction", + "position": 1, + "link": { + "type": "generated-index", + "description": "Learn about ScrapegraphAI, its features, and how it can help you with web scraping." + } +} \ No newline at end of file diff --git a/docs/introduction/contributing.md b/docs/introduction/contributing.md new file mode 100644 index 00000000..7cdeeb77 --- /dev/null +++ b/docs/introduction/contributing.md @@ -0,0 +1,11 @@ +# Contributing + +Hey, you want to contribute? Awesome! +Just fork the repo, make your changes, and send a pull request. +If you're not sure if it's a good idea, open an issue and we'll discuss it. + +Go and check out the [contributing guidelines](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/CONTRIBUTING.md) for more information. + +# License +This project is licensed under the MIT license. +See the [LICENSE](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/LICENSE) file for more details. \ No newline at end of file diff --git a/docs/introduction/diagram.md b/docs/introduction/diagram.md new file mode 100644 index 00000000..e631b924 --- /dev/null +++ b/docs/introduction/diagram.md @@ -0,0 +1,11 @@ +--- +sidebar_position: 4 +--- + +# Library Diagram + +With ScrapegraphAI you can use many already implemented scraping pipelines or create your own. + +The diagram below illustrates the high-level architecture of ScrapeGraphAI: + +![ScrapegraphAI Overview](../../assets/project_overview_diagram.png) \ No newline at end of file diff --git a/docs/introduction/faq.md b/docs/introduction/faq.md new file mode 100644 index 00000000..241f341f --- /dev/null +++ b/docs/introduction/faq.md @@ -0,0 +1,42 @@ +--- +sidebar_position: 5 +--- + +# FAQ + +1. **What is ScrapeGraphAI?** + + ScrapeGraphAI is an open-source python library that uses large language models (LLMs) and graph logic to automate the creation of scraping pipelines for websites and various document types. + +2. **How does ScrapeGraphAI differ from traditional scraping tools?** + + Traditional scraping tools rely on fixed patterns and manual configurations, whereas ScrapeGraphAI adapts to website structure changes using LLMs, reducing the need for constant developer intervention. + +3. **Which LLMs are supported by ScrapeGraphAI?** + + ScrapeGraphAI supports several LLMs, including GPT, Gemini, Groq, Azure, Hugging Face, and local models that can run on your machine using Ollama. + +4. **Can ScrapeGraphAI handle different document formats?** + + Yes, ScrapeGraphAI can scrape information from various document formats such as XML, HTML, JSON, and more. + +5. **I get an empty or incorrect output when scraping a website. What should I do?** + + There are several reasons behind this issue, but for most cases, you can try the following: + + - Set the `headless` parameter to `False` in the graph_config. Some javascript-heavy websites might require it. + - Check your internet connection. Low speed or unstable connection can cause the HTML to not load properly. + - Try using a proxy server to mask your IP address. Check out the [Proxy](../scrapers/proxy.md) section for more information on how to configure proxy settings. + - Use a different LLM model. Some models might perform better on certain websites than others. + - Set the `verbose` parameter to `True` in the graph_config to see more detailed logs. + - Visualize the pipeline graphically using [Burr](../scrapers/burr.md). + + If the issue persists, please report it on the GitHub repository. + +6. **How does ScrapeGraphAI handle the context window limit of LLMs?** + + By splitting big websites/documents into chunks with overlaps and applying compression techniques to reduce the number of tokens. If multiple chunks are present, we will have multiple answers to the user prompt, and therefore, we merge them together in the last step of the scraping pipeline. + +7. **How can I contribute to ScrapeGraphAI?** + + You can contribute to ScrapeGraphAI by submitting bug reports, feature requests, or pull requests on the GitHub repository. Join our [Discord](https://discord.gg/uJN7TYcpNa) community and follow us on social media! \ No newline at end of file diff --git a/docs/introduction/models.md b/docs/introduction/models.md new file mode 100644 index 00000000..c5c1c399 --- /dev/null +++ b/docs/introduction/models.md @@ -0,0 +1,91 @@ +--- +sidebar_position: 3 +--- + +# AI Models and Token Limits + +ScrapGraphAI supports a wide range of AI models from various providers. Each model has a specific token limit, which is important to consider when designing your scraping pipelines. Here's an overview of the supported models and their token limits: + +## OpenAI Models +- GPT-3.5 Turbo (16,385 tokens) +- GPT-3.5 (4,096 tokens) +- GPT-3.5 Turbo Instruct (4,096 tokens) +- GPT-4 Turbo Preview (128,000 tokens) +- GPT-4 Vision Preview (128,000 tokens) +- GPT-4 (8,192 tokens) +- GPT-4 32k (32,768 tokens) +- GPT-4o (128,000 tokens) +- O1 Preview (128,000 tokens) +- O1 Mini (128,000 tokens) + +## Azure OpenAI Models +- GPT-3.5 Turbo (16,385 tokens) +- GPT-3.5 (4,096 tokens) +- GPT-4 Turbo Preview (128,000 tokens) +- GPT-4 (8,192 tokens) +- GPT-4 32k (32,768 tokens) +- GPT-4o (128,000 tokens) +- O1 Preview (128,000 tokens) +- O1 Mini (128,000 tokens) + +## Google AI Models +- Gemini Pro (128,000 tokens) +- Gemini 1.5 Flash (128,000 tokens) +- Gemini 1.5 Pro (128,000 tokens) +- Gemini 1.0 Pro (128,000 tokens) + +## Anthropic Models +- Claude Instant (100,000 tokens) +- Claude 2 (9,000 tokens) +- Claude 2.1 (200,000 tokens) +- Claude 3 (200,000 tokens) +- Claude 3.5 (200,000 tokens) +- Claude 3 Opus (200,000 tokens) +- Claude 3 Sonnet (200,000 tokens) +- Claude 3 Haiku (200,000 tokens) + +## Mistral AI Models +- Mistral Large Latest (128,000 tokens) +- Open Mistral Nemo (128,000 tokens) +- Codestral Latest (32,000 tokens) +- Open Mistral 7B (32,000 tokens) +- Open Mixtral 8x7B (32,000 tokens) +- Open Mixtral 8x22B (64,000 tokens) +- Open Codestral Mamba (256,000 tokens) + +## Ollama Models +- Command-R (12,800 tokens) +- CodeLlama (16,000 tokens) +- DBRX (32,768 tokens) +- DeepSeek Coder 33B (16,000 tokens) +- Llama2 Series (4,096 tokens) +- Llama3 Series (8,192-128,000 tokens) +- Mistral Models (32,000-128,000 tokens) +- Mixtral 8x22B Instruct (65,536 tokens) +- Phi3 Series (12,800-128,000 tokens) +- Qwen Series (32,000 tokens) + +## Hugging Face Models +- Grok-1 (8,192 tokens) +- Meta Llama 3 Series (8,192 tokens) +- Google Gemma Series (8,192 tokens) +- Microsoft Phi Series (2,048-131,072 tokens) +- GPT-2 Series (1,024 tokens) +- DeepSeek V2 Series (131,072 tokens) + +## Bedrock Models +- Claude 3 Series (200,000 tokens) +- Llama2 & Llama3 Series (4,096-8,192 tokens) +- Mistral Series (32,768 tokens) +- Titan Embed Text (8,000 tokens) +- Cohere Embed (512 tokens) + +## Fireworks Models +- Llama V2 7B (4,096 tokens) +- Mixtral 8x7B Instruct (4,096 tokens) +- Llama 3.1 Series (131,072 tokens) +- Mixtral MoE Series (65,536 tokens) + +For a complete and up-to-date list of supported models and their token limits, please refer to the API documentation. + +Understanding token limits is crucial for optimizing your scraping tasks. Larger token limits allow for processing more text in a single API call, which can be beneficial for scraping lengthy web pages or documents. \ No newline at end of file diff --git a/docs/introduction/overview.md b/docs/introduction/overview.md new file mode 100644 index 00000000..b060c08e --- /dev/null +++ b/docs/introduction/overview.md @@ -0,0 +1,164 @@ +--- +sidebar_position: 1 +--- + +# Overview + +![ScrapegraphAI](../../assets/scrapegraphai_logo.png) + +ScrapeGraphAI is an **open-source** Python library designed to revolutionize **scraping** tools. +In today's data-intensive digital landscape, this library stands out by integrating **Large Language Models** (LLMs) +and modular **graph-based** pipelines to automate the scraping of data from various sources (e.g., websites, local files etc.). + +Simply specify the information you need to extract, and ScrapeGraphAI handles the rest, providing a more **flexible** and **low-maintenance** solution compared to traditional scraping tools. + +# Why ScrapegraphAI? + +Traditional web scraping tools often rely on fixed patterns or manual configuration to extract data from web pages. +ScrapegraphAI, leveraging the power of LLMs, adapts to changes in website structures, reducing the need for constant developer intervention. +This flexibility ensures that scrapers remain functional even when website layouts change. + +We support many LLMs including **GPT, Gemini, Groq, Azure, Hugging Face** etc. +as well as local models which can run on your machine using **Ollama**. + +# AI Models and Token Limits + +ScrapGraphAI supports a wide range of AI models from various providers. Each model has a specific token limit, which is important to consider when designing your scraping pipelines. Here's an overview of the supported models and their token limits: + +## OpenAI Models +- GPT-3.5 Turbo (16,385 tokens) +- GPT-3.5 (4,096 tokens) +- GPT-3.5 Turbo Instruct (4,096 tokens) +- GPT-4 Turbo Preview (128,000 tokens) +- GPT-4 Vision Preview (128,000 tokens) +- GPT-4 (8,192 tokens) +- GPT-4 32k (32,768 tokens) +- GPT-4o (128,000 tokens) +- O1 Preview (128,000 tokens) +- O1 Mini (128,000 tokens) + +## Azure OpenAI Models +- GPT-3.5 Turbo (16,385 tokens) +- GPT-3.5 (4,096 tokens) +- GPT-4 Turbo Preview (128,000 tokens) +- GPT-4 (8,192 tokens) +- GPT-4 32k (32,768 tokens) +- GPT-4o (128,000 tokens) +- O1 Preview (128,000 tokens) +- O1 Mini (128,000 tokens) + +## Google AI Models +- Gemini Pro (128,000 tokens) +- Gemini 1.5 Flash (128,000 tokens) +- Gemini 1.5 Pro (128,000 tokens) +- Gemini 1.0 Pro (128,000 tokens) + +## Anthropic Models +- Claude Instant (100,000 tokens) +- Claude 2 (9,000 tokens) +- Claude 2.1 (200,000 tokens) +- Claude 3 (200,000 tokens) +- Claude 3.5 (200,000 tokens) +- Claude 3 Opus (200,000 tokens) +- Claude 3 Sonnet (200,000 tokens) +- Claude 3 Haiku (200,000 tokens) + +## Mistral AI Models +- Mistral Large Latest (128,000 tokens) +- Open Mistral Nemo (128,000 tokens) +- Codestral Latest (32,000 tokens) +- Open Mistral 7B (32,000 tokens) +- Open Mixtral 8x7B (32,000 tokens) +- Open Mixtral 8x22B (64,000 tokens) +- Open Codestral Mamba (256,000 tokens) + +## Ollama Models +- Command-R (12,800 tokens) +- CodeLlama (16,000 tokens) +- DBRX (32,768 tokens) +- DeepSeek Coder 33B (16,000 tokens) +- Llama2 Series (4,096 tokens) +- Llama3 Series (8,192-128,000 tokens) +- Mistral Models (32,000-128,000 tokens) +- Mixtral 8x22B Instruct (65,536 tokens) +- Phi3 Series (12,800-128,000 tokens) +- Qwen Series (32,000 tokens) + +## Hugging Face Models +- Grok-1 (8,192 tokens) +- Meta Llama 3 Series (8,192 tokens) +- Google Gemma Series (8,192 tokens) +- Microsoft Phi Series (2,048-131,072 tokens) +- GPT-2 Series (1,024 tokens) +- DeepSeek V2 Series (131,072 tokens) + +## Bedrock Models +- Claude 3 Series (200,000 tokens) +- Llama2 & Llama3 Series (4,096-8,192 tokens) +- Mistral Series (32,768 tokens) +- Titan Embed Text (8,000 tokens) +- Cohere Embed (512 tokens) + +## Fireworks Models +- Llama V2 7B (4,096 tokens) +- Mixtral 8x7B Instruct (4,096 tokens) +- Llama 3.1 Series (131,072 tokens) +- Mixtral MoE Series (65,536 tokens) + +For a complete and up-to-date list of supported models and their token limits, please refer to the API documentation. + +Understanding token limits is crucial for optimizing your scraping tasks. Larger token limits allow for processing more text in a single API call, which can be beneficial for scraping lengthy web pages or documents. + +# Library Diagram + +With ScrapegraphAI you can use many already implemented scraping pipelines or create your own. + +The diagram below illustrates the high-level architecture of ScrapeGraphAI: + +![ScrapegraphAI Overview](../../assets/project_overview_diagram.png) + +# FAQ + +1. **What is ScrapeGraphAI?** + + ScrapeGraphAI is an open-source python library that uses large language models (LLMs) and graph logic to automate the creation of scraping pipelines for websites and various document types. + +2. **How does ScrapeGraphAI differ from traditional scraping tools?** + + Traditional scraping tools rely on fixed patterns and manual configurations, whereas ScrapeGraphAI adapts to website structure changes using LLMs, reducing the need for constant developer intervention. + +3. **Which LLMs are supported by ScrapeGraphAI?** + + ScrapeGraphAI supports several LLMs, including GPT, Gemini, Groq, Azure, Hugging Face, and local models that can run on your machine using Ollama. + +4. **Can ScrapeGraphAI handle different document formats?** + + Yes, ScrapeGraphAI can scrape information from various document formats such as XML, HTML, JSON, and more. + +5. **I get an empty or incorrect output when scraping a website. What should I do?** + + There are several reasons behind this issue, but for most cases, you can try the following: + + - Set the `headless` parameter to `False` in the graph_config. Some javascript-heavy websites might require it. + - Check your internet connection. Low speed or unstable connection can cause the HTML to not load properly. + - Try using a proxy server to mask your IP address. Check out the [Proxy](#proxy) section for more information on how to configure proxy settings. + - Use a different LLM model. Some models might perform better on certain websites than others. + - Set the `verbose` parameter to `True` in the graph_config to see more detailed logs. + - Visualize the pipeline graphically using [Burr](#burr). + + If the issue persists, please report it on the GitHub repository. + +6. **How does ScrapeGraphAI handle the context window limit of LLMs?** + + By splitting big websites/documents into chunks with overlaps and applying compression techniques to reduce the number of tokens. If multiple chunks are present, we will have multiple answers to the user prompt, and therefore, we merge them together in the last step of the scraping pipeline. + +7. **How can I contribute to ScrapeGraphAI?** + + You can contribute to ScrapeGraphAI by submitting bug reports, feature requests, or pull requests on the GitHub repository. Join our [Discord](https://discord.gg/uJN7TYcpNa) community and follow us on social media! + +# Sponsors + +[![Browserbase](../../assets/browserbase_logo.png)](https://www.browserbase.com/) +[![Serp API](../../assets/serp_api_logo.png)](https://serpapi.com?utm_source=scrapegraphai) +[![Stat Proxies](../../assets/transparent_stat.png)](https://dashboard.statproxies.com/?refferal=scrapegraph) +[![Scrapedo](../../assets/scrapedo.png)](https://scrape.do) \ No newline at end of file diff --git a/docs/introduction/sponsors.md b/docs/introduction/sponsors.md new file mode 100644 index 00000000..2140f742 --- /dev/null +++ b/docs/introduction/sponsors.md @@ -0,0 +1,13 @@ +--- +sidebar_position: 6 +--- + +# Sponsors + +[![Browserbase](../../assets/browserbase_logo.png)](https://www.browserbase.com/) + +[![Serp API](../../assets/serp_api_logo.png)](https://serpapi.com?utm_source=scrapegraphai) + +[![Stat Proxies](../../assets/transparent_stat.png)](https://dashboard.statproxies.com/?refferal=scrapegraph) + +[![Scrapedo](../../assets/scrapedo.png)](https://scrape.do) \ No newline at end of file diff --git a/docs/introduction/why.md b/docs/introduction/why.md new file mode 100644 index 00000000..a0bbb3cd --- /dev/null +++ b/docs/introduction/why.md @@ -0,0 +1,12 @@ +--- +sidebar_position: 2 +--- + +# Why ScrapegraphAI? + +Traditional web scraping tools often rely on fixed patterns or manual configuration to extract data from web pages. +ScrapegraphAI, leveraging the power of LLMs, adapts to changes in website structures, reducing the need for constant developer intervention. +This flexibility ensures that scrapers remain functional even when website layouts change. + +We support many LLMs including **GPT, Gemini, Groq, Azure, Hugging Face** etc. +as well as local models which can run on your machine using **Ollama**. \ No newline at end of file diff --git a/docs/modules/_category_.json b/docs/modules/_category_.json new file mode 100644 index 00000000..84aa2335 --- /dev/null +++ b/docs/modules/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Modules", + "position": 20, + "link": { + "type": "generated-index", + "description": "Explore the core modules and packages of ScrapegraphAI." + } +} \ No newline at end of file diff --git a/docs/modules/modules.md b/docs/modules/modules.md new file mode 100644 index 00000000..e08a1899 --- /dev/null +++ b/docs/modules/modules.md @@ -0,0 +1,9 @@ +--- +sidebar_position: 13 +--- + +# API Reference + +This section contains the complete API reference for ScrapegraphAI. The library is organized into several modules, each handling specific functionality. + +For a complete overview of all modules and their functionality, see the [ScrapegraphAI Package](scrapegraphai.md) documentation. \ No newline at end of file diff --git a/docs/modules/scrapegraphai.builders.md b/docs/modules/scrapegraphai.builders.md new file mode 100644 index 00000000..d356ea5f --- /dev/null +++ b/docs/modules/scrapegraphai.builders.md @@ -0,0 +1,23 @@ +--- +sidebar_position: 2 +--- + +# scrapegraphai.builders package + +## Graph Builders + +### Basic Builders + +Functions for building basic graph structures: + +```python +from scrapegraphai.builders import build_basic_graph, build_multi_graph +``` + +### Specialized Builders + +Functions for building specialized graph types: + +```python +from scrapegraphai.builders import build_search_graph, build_script_graph +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.docloaders.md b/docs/modules/scrapegraphai.docloaders.md new file mode 100644 index 00000000..b9f903bd --- /dev/null +++ b/docs/modules/scrapegraphai.docloaders.md @@ -0,0 +1,23 @@ +--- +sidebar_position: 5 +--- + +# scrapegraphai.docloaders package + +## Document Loaders + +### Web Loaders + +Functions for loading web content: + +```python +from scrapegraphai.docloaders import WebLoader, HTMLLoader +``` + +### File Loaders + +Functions for loading local files: + +```python +from scrapegraphai.docloaders import FileLoader, PDFLoader, TextLoader +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.graphs.md b/docs/modules/scrapegraphai.graphs.md new file mode 100644 index 00000000..a1a4f6ab --- /dev/null +++ b/docs/modules/scrapegraphai.graphs.md @@ -0,0 +1,39 @@ +--- +sidebar_position: 3 +--- + +# scrapegraphai.graphs package + +## Available Graphs + +### Basic Scrapers + +The basic scraping graphs for single and multi-source extraction: + +```python +from scrapegraphai.graphs import SmartScraperGraph, SmartScraperMultiGraph +``` + +### Search-based Scrapers + +Graphs that combine search capabilities with scraping: + +```python +from scrapegraphai.graphs import SearchGraph, OmniSearchGraph +``` + +### Specialized Scrapers + +Graphs for specific use cases like script generation and speech: + +```python +from scrapegraphai.graphs import ScriptCreatorGraph, SpeechGraph +``` + +### Advanced Scrapers + +Advanced graphs with additional capabilities: + +```python +from scrapegraphai.graphs import OmniScraperGraph +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.helpers.md b/docs/modules/scrapegraphai.helpers.md new file mode 100644 index 00000000..ea9f2247 --- /dev/null +++ b/docs/modules/scrapegraphai.helpers.md @@ -0,0 +1,39 @@ +--- +sidebar_position: 4 +--- + +# scrapegraphai.helpers package + +## Helper Functions + +### Model Tokens + +Functions for managing model tokens and limits: + +```python +from scrapegraphai.helpers.models_tokens import get_model_tokens, validate_tokens +``` + +### Configuration + +Functions for handling configuration settings: + +```python +from scrapegraphai.helpers import load_config, validate_config +``` + +### Error Handling + +Functions for consistent error handling: + +```python +from scrapegraphai.helpers import handle_error, format_error +``` + +### Logging + +Functions for logging and debugging: + +```python +from scrapegraphai.helpers import setup_logging, log_debug +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.integrations.md b/docs/modules/scrapegraphai.integrations.md new file mode 100644 index 00000000..55632fad --- /dev/null +++ b/docs/modules/scrapegraphai.integrations.md @@ -0,0 +1,31 @@ +--- +sidebar_position: 9 +--- + +# scrapegraphai.integrations package + +## Available Integrations + +### Search Engines + +Integrations with various search engines: + +```python +from scrapegraphai.integrations import GoogleSearch, BingSearch +``` + +### Browser Automation + +Integrations for browser automation: + +```python +from scrapegraphai.integrations import PlaywrightBrowser, SeleniumBrowser +``` + +### Proxy Services + +Integrations with proxy services: + +```python +from scrapegraphai.integrations import ProxyRotator, ProxyManager +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.md b/docs/modules/scrapegraphai.md new file mode 100644 index 00000000..c9824ea5 --- /dev/null +++ b/docs/modules/scrapegraphai.md @@ -0,0 +1,50 @@ +--- +sidebar_position: 1 +--- + +# scrapegraphai + +The main package containing all ScrapegraphAI functionality. + +## Overview + +ScrapegraphAI is organized into several submodules, each handling specific aspects of the scraping pipeline: + +### Core Functionality +- [scrapegraphai.graphs](scrapegraphai.graphs.md) - Graph-based scraping pipelines +- [scrapegraphai.models](scrapegraphai.models.md) - LLM and embedding model implementations +- [scrapegraphai.nodes](scrapegraphai.nodes.md) - Building blocks for scraping graphs + +### Support Modules +- [scrapegraphai.builders](scrapegraphai.builders.md) - Graph construction utilities +- [scrapegraphai.docloaders](scrapegraphai.docloaders.md) - Document loading and processing +- [scrapegraphai.helpers](scrapegraphai.helpers.md) - Helper functions and utilities +- [scrapegraphai.integrations](scrapegraphai.integrations.md) - Third-party service integrations +- [scrapegraphai.utils](scrapegraphai.utils.md) - General utility functions + +## Installation + +```bash +pip install scrapegraphai +``` + +## Basic Usage + +```python +from scrapegraphai.graphs import SmartScraperGraph + +# Create a scraper instance +scraper = SmartScraperGraph( + prompt="Extract product information", + source="https://example.com/products", + config={ + "llm": { + "model": "gpt-3.5-turbo" + } + } +) + +# Run the scraper +result = scraper.run() +print(result) +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.models.md b/docs/modules/scrapegraphai.models.md new file mode 100644 index 00000000..0f9ce945 --- /dev/null +++ b/docs/modules/scrapegraphai.models.md @@ -0,0 +1,39 @@ +--- +sidebar_position: 6 +--- + +# scrapegraphai.models package + +## Submodules + +### Base Models + +The base module provides abstract base classes and interfaces for all model types in the library. + +```python +from scrapegraphai.models.base import BaseModel +``` + +### Embeddings Models + +The embeddings module contains implementations for various text embedding models. + +```python +from scrapegraphai.models.embeddings import OpenAIEmbeddings, OllamaEmbeddings +``` + +### LLM Models + +The LLM module provides implementations for different large language models. + +```python +from scrapegraphai.models.llm import OpenAIChat, OllamaLLM, GeminiChat +``` + +### Text-to-Speech Models + +The TTS module contains implementations for text-to-speech models. + +```python +from scrapegraphai.models.tts import OpenAITTS +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.nodes.md b/docs/modules/scrapegraphai.nodes.md new file mode 100644 index 00000000..ecb62122 --- /dev/null +++ b/docs/modules/scrapegraphai.nodes.md @@ -0,0 +1,39 @@ +--- +sidebar_position: 7 +--- + +# scrapegraphai.nodes package + +## Graph Nodes + +### Base Nodes + +The fundamental building blocks for creating graph nodes: + +```python +from scrapegraphai.nodes import BaseNode, InputNode, OutputNode +``` + +### Processing Nodes + +Nodes for processing and transforming data: + +```python +from scrapegraphai.nodes import TextProcessingNode, HTMLProcessingNode +``` + +### LLM Nodes + +Nodes that interact with language models: + +```python +from scrapegraphai.nodes import LLMNode, EmbeddingNode +``` + +### Specialized Nodes + +Nodes for specific tasks: + +```python +from scrapegraphai.nodes import SearchNode, ScriptGeneratorNode, TTSNode +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai.utils.md b/docs/modules/scrapegraphai.utils.md new file mode 100644 index 00000000..f77ae784 --- /dev/null +++ b/docs/modules/scrapegraphai.utils.md @@ -0,0 +1,47 @@ +--- +sidebar_position: 8 +--- + +# scrapegraphai.utils package + +## Utility Functions + +### Execution Info + +Functions for handling execution information and formatting: + +```python +from scrapegraphai.utils import prettify_exec_info, get_exec_info +``` + +### HTML Processing + +Functions for processing and cleaning HTML content: + +```python +from scrapegraphai.utils import clean_html, extract_text_from_html +``` + +### Token Management + +Functions for managing and counting tokens: + +```python +from scrapegraphai.utils import count_tokens, split_text_into_chunks +``` + +### Schema Handling + +Functions for handling JSON schemas and validation: + +```python +from scrapegraphai.utils import validate_schema, generate_schema +``` + +### File Operations + +Functions for handling file operations: + +```python +from scrapegraphai.utils import read_file, write_file, download_file +``` \ No newline at end of file diff --git a/docs/modules/scrapegraphai_package.md b/docs/modules/scrapegraphai_package.md new file mode 100644 index 00000000..5b222647 --- /dev/null +++ b/docs/modules/scrapegraphai_package.md @@ -0,0 +1,34 @@ +--- +sidebar_position: 2 +--- + +# scrapegraphai package + +## Package Structure + +ScrapegraphAI is organized into several key packages: + +``` +scrapegraphai/ +โ”œโ”€โ”€ graphs/ # Graph-based scraping pipelines +โ”œโ”€โ”€ models/ # LLM and embedding implementations +โ”œโ”€โ”€ nodes/ # Graph building blocks +โ”œโ”€โ”€ builders/ # Graph construction utilities +โ”œโ”€โ”€ docloaders/ # Document loading and processing +โ”œโ”€โ”€ helpers/ # Helper functions +โ”œโ”€โ”€ integrations/ # Third-party integrations +โ””โ”€โ”€ utils/ # Utility functions +``` + +Each package serves a specific purpose in the scraping pipeline: + +- **graphs**: Contains the main scraping pipeline implementations +- **models**: Provides interfaces to various LLM and embedding models +- **nodes**: Defines the building blocks used to construct scraping graphs +- **builders**: Offers utilities for constructing custom graphs +- **docloaders**: Handles loading and processing of various document types +- **helpers**: Contains helper functions used throughout the library +- **integrations**: Manages integrations with third-party services +- **utils**: Provides general utility functions + +For detailed information about each package, see their respective documentation pages. \ No newline at end of file diff --git a/docs/scrapers/_category_.json b/docs/scrapers/_category_.json new file mode 100644 index 00000000..e048274f --- /dev/null +++ b/docs/scrapers/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Scrapers", + "position": 3, + "link": { + "type": "generated-index", + "description": "Learn about the different types of scrapers and their configurations in ScrapeGraphAI." + } +} \ No newline at end of file diff --git a/docs/scrapers/configuration.md b/docs/scrapers/configuration.md new file mode 100644 index 00000000..9fd1ad88 --- /dev/null +++ b/docs/scrapers/configuration.md @@ -0,0 +1,84 @@ +--- +sidebar_position: 4 +--- + +# Additional Parameters + +It is possible to customize the behavior of the graphs by setting some configuration options. +Some interesting ones are: + +- `verbose`: If set to `True`, some debug information will be printed to the console. +- `headless`: If set to `False`, the web browser will be opened on the URL requested and close right after the HTML is fetched. +- `max_results`: The maximum number of results to be fetched from the search engine. Useful in `SearchGraph`. +- `output_path`: The path where the output files will be saved. Useful in `SpeechGraph`. +- `loader_kwargs`: A dictionary with additional parameters to be passed to the `Loader` class, such as `proxy`. +- `burr_kwargs`: A dictionary with additional parameters to enable `Burr` graphical user interface. +- `max_images`: The maximum number of images to be analyzed. Useful in `OmniScraperGraph` and `OmniSearchGraph`. +- `cache_path`: The path where the cache files will be saved. If already exists, the cache will be loaded from this path. +- `additional_info`: Add additional text to default prompts defined in the graphs. + +## Burr Integration + +`Burr` is an open source python library that allows the creation and management of state machine applications. Discover more about it [here](https://github.com/DAGWorks-Inc/burr). +It is possible to enable a local hosted webapp to visualize the scraping pipelines and the data flow. +First, we need to install the `burr` library as follows: + +```bash +pip install scrapegraphai[burr] +``` + +and then run the graphical user interface as follows: + +```bash +burr +``` + +To log your graph execution in the platform, you need to set the `burr_kwargs` parameter in the graph configuration as follows: + +```python +graph_config = { + "llm":{...}, + "burr_kwargs": { + "project_name": "test-scraper", + "app_instance_id":"some_id", + } +} +``` + +## Proxy Rotation + +It is possible to rotate the proxy by setting the `proxy` option in the graph configuration. +We provide a free proxy service which is based on [free-proxy](https://pypi.org/project/free-proxy/) library and can be used as follows: + +```python +graph_config = { + "llm":{...}, + "loader_kwargs": { + "proxy" : { + "server": "broker", + "criteria": { + "anonymous": True, + "secure": True, + "countryset": {"IT"}, + "timeout": 10.0, + "max_shape": 3 + }, + }, + }, +} +``` + +Do you have a proxy server? You can use it as follows: + +```python +graph_config = { + "llm":{...}, + "loader_kwargs": { + "proxy" : { + "server": "http://your_proxy_server:port", + "username": "your_username", + "password": "your_password", + }, + }, +} +``` \ No newline at end of file diff --git a/docs/scrapers/graphs.md b/docs/scrapers/graphs.md new file mode 100644 index 00000000..b2f44d6f --- /dev/null +++ b/docs/scrapers/graphs.md @@ -0,0 +1,14 @@ +--- +sidebar_position: 1 +--- + +# Graphs + +Graphs are scraping pipelines aimed at solving specific tasks. They are composed by nodes which can be configured individually to address different aspects of the task (fetching data, extracting information, etc.). + +In this section, you'll find information about: +- Different types of graphs and their use cases +- LLM configuration and supported models +- Graph configuration options +- Benchmarks and performance metrics +- Telemetry and monitoring \ No newline at end of file diff --git a/docs/scrapers/telemetry.md b/docs/scrapers/telemetry.md new file mode 100644 index 00000000..cd68a74a --- /dev/null +++ b/docs/scrapers/telemetry.md @@ -0,0 +1,80 @@ +--- +sidebar_position: 5 +--- + +# Usage Analytics + +ScrapeGraphAI collects **anonymous** usage data by default to improve the library and guide development efforts. + +## Events Captured + +We capture events in the following scenarios: + +1. When a `Graph` finishes running. +2. When an exception is raised in one of the nodes. + +## Data Collected + +The data captured is limited to: + +- Operating System and Python version +- A persistent UUID to identify the session, stored in `~/.scrapegraphai.conf` + +Additionally, the following properties are collected: + +```python +properties = { + "graph_name": graph_name, + "llm_model": llm_model_name, + "embedder_model": embedder_model_name, + "source_type": source_type, + "source": source, + "execution_time": execution_time, + "prompt": prompt, + "schema": schema, + "error_node": error_node_name, + "exception": exception, + "response": response, + "total_tokens": total_tokens, +} +``` + +For more details, refer to the [telemetry.py](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/scrapegraphai/telemetry/telemetry.py) module. + +## Opting Out + +If you prefer not to participate in telemetry, you can opt out using any of the following methods: + +1. **Programmatically Disable Telemetry**: + + Add the following code at the beginning of your script: + + ```python + from scrapegraphai import telemetry + telemetry.disable_telemetry() + ``` + +2. **Configuration File**: + + Set the `telemetry_enabled` key to `false` in `~/.scrapegraphai.conf` under the `[DEFAULT]` section: + + ```ini + [DEFAULT] + telemetry_enabled = False + ``` + +3. **Environment Variable**: + + - **For a Shell Session**: + + ```bash + export SCRAPEGRAPHAI_TELEMETRY_ENABLED=false + ``` + + - **For a Single Command**: + + ```bash + SCRAPEGRAPHAI_TELEMETRY_ENABLED=false python my_script.py + ``` + +By following any of these methods, you can easily opt out of telemetry and ensure your usage data is not collected. \ No newline at end of file diff --git a/docusaurus.config.js b/docusaurus.config.js index 90035cb0..9fa9324d 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -49,7 +49,11 @@ const config = { // Remove this to remove the "edit this page" links. editUrl: 'https://github.com/ScrapeGraphAI/docusaurus-docs', + routeBasePath: '/', // Make docs the default landing page + path: 'docs', + sidebarCollapsed: false, }, + blog: false, // Disable blog theme: { customCss: './src/css/custom.css', }, @@ -79,6 +83,7 @@ const config = { sidebarId: 'tutorialSidebar', position: 'left', label: 'Documentation', + to: '/', }, { href: 'https://scrapegraphai.com', @@ -86,7 +91,7 @@ const config = { position: 'left', }, { - href: 'https://github.com/VinciGit00/Scrapegraph-ai', + href: 'https://github.com/ScrapeGraphAI/Scrapegraph-ai', label: 'GitHub', position: 'right', }, @@ -100,7 +105,11 @@ const config = { items: [ { label: 'Tutorial', - to: '/docs/intro', + to: '/', + }, + { + label: 'Official website', + href: 'https://scrapegraphai.com', }, ], }, @@ -114,7 +123,7 @@ const config = { ], }, ], - copyright: `Copyright ยฉ ${new Date().getFullYear()}, Marco Perini, Lorenzo Padoan e Marco Vinciguerra. Built with Docusaurus.`, + copyright: `Copyright ยฉ ${new Date().getFullYear()}, ScrapegraphAI, Inc. Built with Docusaurus.`, }, prism: { theme: prismThemes.github, diff --git a/src/pages/index.js b/src/pages/index.js deleted file mode 100644 index 7909830f..00000000 --- a/src/pages/index.js +++ /dev/null @@ -1,37 +0,0 @@ -import clsx from 'clsx'; -import Link from '@docusaurus/Link'; -import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; -import Layout from '@theme/Layout'; -import HomepageFeatures from '@site/src/components/HomepageFeatures'; - -import Heading from '@theme/Heading'; -import styles from './index.module.css'; - -function HomepageHeader() { - const {siteConfig} = useDocusaurusContext(); - return ( -
-
- - {siteConfig.title} - -

{siteConfig.tagline}

-

You only scrape once

-
-
- ); -} - -export default function Home() { - const {siteConfig} = useDocusaurusContext(); - return ( - - -
- -
- - ); -} diff --git a/src/pages/index.module.css b/src/pages/index.module.css deleted file mode 100644 index 9f71a5da..00000000 --- a/src/pages/index.module.css +++ /dev/null @@ -1,23 +0,0 @@ -/** - * CSS files with the .module.css suffix will be treated as CSS modules - * and scoped locally. - */ - -.heroBanner { - padding: 4rem 0; - text-align: center; - position: relative; - overflow: hidden; -} - -@media screen and (max-width: 996px) { - .heroBanner { - padding: 2rem; - } -} - -.buttons { - display: flex; - align-items: center; - justify-content: center; -}