From d3f729f368d5a7337f8f4af4042da314e0cebe74 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Oct 2025 09:18:48 -0700 Subject: [PATCH 01/33] Update --- docs/source/conf.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 760a8d714..2d1c95e6a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -157,15 +157,6 @@ def get_version_path(): "html_image", ] -autodoc_default_options = { - "members": True, - "member-order": "bysource", - "special-members": "__init__", - "undoc-members": True, - "exclude-members": "__weakref__", -} - - # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { "examples_dirs": "tutorial_sources", # Path to examples directory @@ -179,3 +170,4 @@ def get_version_path(): "write_computation_times": True, "show_signature": False, } + From 9823f1f12fb09f0284e894104d03d000e0da0b68 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Oct 2025 09:19:54 -0700 Subject: [PATCH 02/33] Update --- docs/source/api.md | 53 +- docs/source/api_actors.md | 55 +- docs/source/api_controller.md | 3 - docs/source/api_core.md | 69 +- docs/source/api_data.md | 67 +- docs/source/api_data_models.md | 51 ++ docs/source/api_envs.md | 8 - docs/source/api_losses.md | 26 +- docs/source/api_types.md | 41 + docs/source/api_util.md | 86 +- ...r_rl_fundamentals_forge_tutorial_thumb.png | Bin 0 -> 28056 bytes .../sphx_glr_template_tutorial_thumb.png | Bin 0 -> 28056 bytes docs/source/tutorials/index.rst | 63 ++ ...l_fundamentals_forge_tutorial.codeobj.json | 215 +++++ .../rl_fundamentals_forge_tutorial.ipynb | 158 ++++ .../rl_fundamentals_forge_tutorial.py | 558 +++++++++++++ .../rl_fundamentals_forge_tutorial.py.md5 | 1 + .../rl_fundamentals_forge_tutorial.rst | 788 ++++++++++++++++++ .../rl_fundamentals_forge_tutorial.zip | Bin 0 -> 40960 bytes docs/source/tutorials/sg_execution_times.rst | 40 + .../tutorials/template_tutorial.codeobj.json | 27 + docs/source/tutorials/template_tutorial.ipynb | 75 ++ docs/source/tutorials/template_tutorial.py | 91 ++ .../source/tutorials/template_tutorial.py.md5 | 1 + docs/source/tutorials/template_tutorial.rst | 152 ++++ docs/source/tutorials/template_tutorial.zip | Bin 0 -> 5757 bytes 26 files changed, 2552 insertions(+), 76 deletions(-) delete mode 100644 docs/source/api_controller.md create mode 100644 docs/source/api_data_models.md delete mode 100644 docs/source/api_envs.md create mode 100644 docs/source/api_types.md create mode 100644 docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png create mode 100644 docs/source/tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png create mode 100644 docs/source/tutorials/index.rst create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.rst create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.zip create mode 100644 docs/source/tutorials/sg_execution_times.rst create mode 100644 docs/source/tutorials/template_tutorial.codeobj.json create mode 100644 docs/source/tutorials/template_tutorial.ipynb create mode 100644 docs/source/tutorials/template_tutorial.py create mode 100644 docs/source/tutorials/template_tutorial.py.md5 create mode 100644 docs/source/tutorials/template_tutorial.rst create mode 100644 docs/source/tutorials/template_tutorial.zip diff --git a/docs/source/api.md b/docs/source/api.md index 5ed009c4c..e4c2d064e 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -1,35 +1,34 @@ # API Reference -This section provides comprehensive API documentation for TorchForge modules and classes. - -TorchForge is organized into several key modules, each providing specialized functionality for post-training generative AI models: - -## Module Overview +This API reference is organized by priority of concepts that users should be exposed to, based on how they're used in the core Forge workflows. + +```{eval-rst} +.. toctree:: + :maxdepth: 2 + + api_core + api_actors + api_data_models + api_types + api_util + api_data + api_losses +``` -**Core Components** -- [Interfaces & Types](api_core.md) - Core interfaces and type definitions -- [Actors](api_actors.md) - Model training and inference components -- [Controller](api_controller.md) - Distributed training orchestration and resource management +## Overview -**Data Management** -- [Data](api_data.md) - Data handling utilities, datasets, and data models +The Forge API is structured around key concepts in order of priority: -**Training Components** -- [Losses](api_losses.md) - Loss functions for reinforcement learning and supervised fine-tuning -- [Environments](api_envs.md) - Training and inference environments +1. **[Core Concepts](api_core.md)** - Actor System, ForgeActor, and ForgeService fundamentals +2. **[Built-in Actors](api_actors.md)** - Policy, Trainer, ReplayBuffer, and ReferenceModel +3. **[Data Models](api_data_models.md)** - Completion, Prompt, Episode, and other data structures +4. **[Configuration and Types](api_types.md)** - Core types and configuration classes +5. **[Utilities](api_util.md)** - Distributed computing, logging, and observability tools +6. **[Data Processing](api_data.md)** - Rewards, tokenization, and data handling utilities +7. **[Loss Functions](api_losses.md)** - GRPO and REINFORCE loss implementations -**Tools & Utilities** -- [Utilities](api_util.md) - General utility functions and helpers +## Quick Start -```{toctree} -:maxdepth: 2 -:hidden: +To get started with Forge, begin with the [Core Concepts](api_core.md) to understand the actor system foundation, then explore the [Built-in Actors](api_actors.md) for common RL workflows. -api_core -api_actors -api_data -api_losses -api_envs -api_controller -api_util -``` +For a practical example, see the GRPO implementation in `apps/grpo/main.py` which demonstrates how these components work together in a complete reinforcement learning training loop. diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index 6ef5f1ff8..1be09712d 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -1,19 +1,54 @@ -# Actors +# Built-in Actors -The actors module contains the core components for model training and inference in TorchForge. This includes policy actors, reference models, replay buffers, and trainers. +On top of the services/actors foundation, Forge provides implementations of actors that are useful in RL workflows. -## Policy Actor +## Policy -The policy actor is responsible for model inference and policy interactions during training. +Inference and generation via vLLM. The {class}`forge.actors.policy.Policy` is a key actor for generating completions from language models. -## Reference Model +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: -The reference model provides baseline comparisons for reinforcement learning algorithms. + forge.actors.policy.Policy + forge.actors.policy.PolicyWorker + forge.actors.policy.EngineConfig + forge.actors.policy.SamplingConfig +``` -## Replay Buffer +## Trainer -The replay buffer manages storage and sampling of training experiences. +Training via torchtitan. The {class}`forge.actors.trainer.RLTrainer` handles reinforcement learning training loops. -## Trainer +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.trainer.RLTrainer +``` + +## ReplayBuffer + +For storing experience and sampling to the trainer - the glue between policy and trainer. The {class}`forge.actors.replay_buffer.ReplayBuffer` manages experience data for RL training. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.replay_buffer.ReplayBuffer +``` + +## ReferenceModel + +Used for RL correctness. The {class}`forge.actors.reference_model.ReferenceModel` provides reference logprobs for RL algorithms. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: -The trainer orchestrates the training process and implements training algorithms. + forge.actors.reference_model.ReferenceModel +``` diff --git a/docs/source/api_controller.md b/docs/source/api_controller.md deleted file mode 100644 index e9bedda74..000000000 --- a/docs/source/api_controller.md +++ /dev/null @@ -1,3 +0,0 @@ -# Controller - -Distributed training orchestration and resource management components for TorchForge. diff --git a/docs/source/api_core.md b/docs/source/api_core.md index 75b3e9ae5..785302ad3 100644 --- a/docs/source/api_core.md +++ b/docs/source/api_core.md @@ -1,3 +1,68 @@ -# Core Interfaces +# Core Concepts -This section covers the fundamental interfaces and type definitions that form the foundation of TorchForge. +## Actor System + +Forge is built on top of Monarch and makes extensive use of actors. +Actors are the foundation for building distributed, fault-tolerant systems. + +## ForgeActor + +In Forge, everything centers around the {class}`forge.controller.actor.ForgeActor`, which is a customized version of a Monarch actor tailored for Forge-specific needs. + +The {class}`forge.controller.actor.ForgeActor` differs from a standard Monarch actor by allowing you to specify resource requirements using the `options()` method. This API lets you define the resources your actor needs and create two types of constructs: + +- A regular Monarch actor using `as_actor()` +- A service using `as_service()` + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.controller.actor.ForgeActor +``` + +### Resource Configuration Example + +Options are important because they demonstrate the resource requirements and how we represent them in Forge: + +```python +from forge.controller.actor import ForgeActor + +class MyActor(ForgeActor): + pass + +# Create a service with specific resource requirements +service = MyActor.options( + hosts=1, + procs=8, + replicas=1, + with_gpus=True +).as_service() +``` + +This example creates a service that has 1 replica, where the replica consists of 1 remote host and 8 processes, using Monarch's remote allocations. + +### Key Methods + +**`options()`** +Configures resource requirements for the actor. + +**`setup()`** +Sets up an actor. All actors should implement this for any heavyweight setup (like PyTorch distributed initialization, model checkpoint loading, etc.). + +**`launch()`** +Logic to provision and deploy a new replica. This is what services use to spin up replicas. + +## ForgeService + +Services are replicated, fault-tolerant versions of actors. They provide high availability and load distribution across multiple actor instances. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.controller.service.Service + forge.controller.service.ServiceInterface +``` diff --git a/docs/source/api_data.md b/docs/source/api_data.md index cbc1cfc53..a652c6d0a 100644 --- a/docs/source/api_data.md +++ b/docs/source/api_data.md @@ -1,16 +1,63 @@ -# Data Management +# Data Processing -Comprehensive data handling utilities for training and -inference, including datasets, data models, and various -data processing utilities. +Data handling utilities for datasets, rewards, and tokenization. -## Prompt +## Rewards -Data model for input prompts and contexts. +Reward functions for RL training. The {mod}`forge.data.rewards` module provides reward functions like {class}`forge.data.rewards.MathReward` and {class}`forge.data.rewards.ThinkingReward`. ```{eval-rst} -.. automodule:: forge.data_models.prompt - :members: - :undoc-members: - :show-inheritance: +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.rewards +``` + +## Tokenization + +Tokenization utilities for processing text data. The {mod}`forge.data.tokenizer` module provides tokenization functions and utilities. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.tokenizer +``` + +## Data Collation + +Data collation utilities for batching and processing. The {mod}`forge.data.collate` module provides functions for collating data into batches. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.collate +``` + +## Data Sharding + +Data sharding utilities for distributed processing. The {mod}`forge.data.sharding` module provides sharding strategies for distributed training. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.sharding +``` + +## Data Utilities + +General data processing utilities. The {mod}`forge.data.utils` module provides miscellaneous data handling functions. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.utils ``` diff --git a/docs/source/api_data_models.md b/docs/source/api_data_models.md new file mode 100644 index 000000000..a7c02caa0 --- /dev/null +++ b/docs/source/api_data_models.md @@ -0,0 +1,51 @@ +# Data Models + +Base data models for common RL workflows. + +## Completion + +Outputs from vLLM. The {class}`forge.data_models.completion.Completion` represents a model-generated completion for a given prompt. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.completion.Completion +``` + +## Prompt + +Input prompts for models. The {class}`forge.data_models.prompt.Prompt` encapsulates prompt data for language models. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.prompt.Prompt +``` + +## Episode + +Training episodes for RL. The {class}`forge.data_models.episode.Episode` represents a complete interaction episode in reinforcement learning. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.episode.Episode +``` + +## ScoredCompletion + +Completions with associated scores. The {class}`forge.data_models.scored_completion.ScoredCompletion` extends completions with scoring information. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.scored_completion.ScoredCompletion +``` diff --git a/docs/source/api_envs.md b/docs/source/api_envs.md deleted file mode 100644 index 88e9d1cea..000000000 --- a/docs/source/api_envs.md +++ /dev/null @@ -1,8 +0,0 @@ -# Environments - -Training and inference environments for TorchForge models. - - -## Chat Environment - -Chat-based environment for conversational AI training and inference. diff --git a/docs/source/api_losses.md b/docs/source/api_losses.md index 097b83394..d16f2a6fc 100644 --- a/docs/source/api_losses.md +++ b/docs/source/api_losses.md @@ -1,11 +1,27 @@ -# Losses +# Loss Functions -Loss functions for reinforcement learning and supervised fine-tuning in TorchForge. +Loss functions for RL training. ## GRPO Loss -Generalized Reward Policy Optimization (GRPO) loss implementation for reinforcement learning. +Group Relative Policy Optimization loss function. The {mod}`forge.losses.grpo_loss` module provides loss functions for GRPO training. -## Reinforce Loss +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: -Reinforce algorithm loss implementation for policy gradient methods. + forge.losses.grpo_loss +``` + +## REINFORCE Loss + +REINFORCE algorithm loss function. The {mod}`forge.losses.reinforce_loss` module provides loss functions for REINFORCE training. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.losses.reinforce_loss +``` diff --git a/docs/source/api_types.md b/docs/source/api_types.md new file mode 100644 index 000000000..e7bd423a3 --- /dev/null +++ b/docs/source/api_types.md @@ -0,0 +1,41 @@ +# Configuration and Types + +## Core Type Definitions + +Core type definitions used throughout Forge. The {mod}`forge.types` module contains fundamental types and configurations. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.types +``` + +## Configuration Classes + +Configuration classes for actors and services. + +### EngineConfig + +Configuration for vLLM engines used in Policy actors. The {class}`forge.actors.policy.EngineConfig` extends vLLM's EngineArgs with worker-specific fields. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.policy.EngineConfig +``` + +### SamplingConfig + +Configuration for sampling parameters in Policy actors. The {class}`forge.actors.policy.SamplingConfig` provides overrides for vLLM's sampling parameters. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.policy.SamplingConfig +``` diff --git a/docs/source/api_util.md b/docs/source/api_util.md index f15e03b76..06438b358 100644 --- a/docs/source/api_util.md +++ b/docs/source/api_util.md @@ -1,25 +1,89 @@ # Utilities -General utility functions and helpers used throughout TorchForge. +Utility functions and classes for distributed computing, logging, and operations. ## Distributed Computing -Utilities for distributed training and communication. +Utilities for distributed computing operations. The {mod}`forge.util.distributed` module provides functions for setting up distributed environments. ```{eval-rst} -.. automodule:: forge.util.distributed - :members: - :undoc-members: - :show-inheritance: +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.distributed ``` ## Logging -Logging configuration and utilities. +Logging utilities for Forge applications. The {mod}`forge.util.logging` module provides logging configuration and utilities. ```{eval-rst} -.. automodule:: forge.util.logging - :members: - :undoc-members: - :show-inheritance: +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.logging +``` + +## Operations + +Core operations and utilities. The {mod}`forge.util.ops` module contains commonly used operations for RL workflows. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.ops +``` + +## Metric Logging + +Metric logging utilities. The {mod}`forge.util.metric_logging` module provides utilities for logging metrics. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.metric_logging +``` + +## Observability + +### Metrics + +Metrics collection and reporting. The {mod}`forge.observability.metrics` module provides the core metrics infrastructure. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.observability.metrics +``` + +### Performance Tracking + +Performance tracking utilities. The {mod}`forge.observability.perf_tracker` module provides performance measurement tools. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.observability.perf_tracker +``` + +### Metric Actors + +Actors for metric collection. The {mod}`forge.observability.metric_actors` module provides actors for distributed metric collection. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.observability.metric_actors ``` diff --git a/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png b/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png new file mode 100644 index 0000000000000000000000000000000000000000..3b362cf91299659eeec19a203a6d16971bc138fd GIT binary patch literal 28056 zcmd3NV|OM^w03Me6XT9;+qP}nwr$(CHNnKTZF@4&o9C?a>HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C&HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C& + +.. thumbnail-parent-div-open + +.. raw:: html + +
+ +.. only:: html + + .. image:: /tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png + :alt: + + :ref:`sphx_glr_tutorials_template_tutorial.py` + +.. raw:: html + +
Template Tutorial
+
+ + +.. raw:: html + +
+ +.. only:: html + + .. image:: /tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png + :alt: + + :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` + +.. raw:: html + +
RL Fundamentals Using Forge Terminology
+
+ + +.. thumbnail-parent-div-close + +.. raw:: html + + + + +.. toctree:: + :hidden: + + /tutorials/template_tutorial + /tutorials/rl_fundamentals_forge_tutorial + diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json b/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json new file mode 100644 index 000000000..c1c95a0b1 --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json @@ -0,0 +1,215 @@ +{ + "Any": [ + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "_SpecialForm" + }, + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "Any" + } + ], + "Dict": [ + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "_SpecialGenericAlias" + }, + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "Dict" + } + ], + "MockResponse": [ + { + "is_class": true, + "is_explicit": false, + "module": "__main__", + "module_short": "__main__", + "name": "MockResponse" + } + ], + "Optional": [ + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "_SpecialForm" + }, + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "Optional" + } + ], + "asyncio.gather": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + }, + { + "is_class": false, + "is_explicit": false, + "module": "asyncio", + "module_short": "asyncio", + "name": "gather" + } + ], + "conceptual_rl_step": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_advantages_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_dataset_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_episode_from_response": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_policy_service": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_reference_model_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_replay_buffer_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_reward_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_trainer_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "demonstrate_production_scaling": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "example_experience": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "dict" + } + ], + "show_infrastructure_challenges": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "torch.cat": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "builtin_function_or_method" + }, + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "cat" + } + ], + "torch.tensor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "builtin_function_or_method" + }, + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "tensor" + } + ] +} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb b/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb new file mode 100644 index 000000000..3981a531b --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n# RL Fundamentals Using Forge Terminology\n\n**Author:** [Your Name](https://github.com/yourusername)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Core RL components in Forge (Dataset, Policy, Reward Model, etc.)\n * How RL concepts map to distributed Forge services\n * Building scalable RL training loops with fault tolerance\n * Resource management and independent scaling patterns\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0+\n * GPU access recommended\n * Basic understanding of reinforcement learning\n * Familiarity with async/await in Python\n\nThis tutorial teaches RL fundamentals using Forge's exact terminology and architecture.\nWe'll start with a simple math tutoring example to understand how traditional RL concepts\nmap to Forge's distributed service model.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Core RL Components in Forge\n\nLet's start with a simple math tutoring example to understand RL concepts\nwith the exact names Forge uses. Think of it as teaching an AI student:\n\n- **Dataset**: Provides questions (like \"What is 2+2?\")\n- **Policy**: The AI student being trained (generates answers like \"The answer is 4\")\n- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95)\n- **Reference Model**: Copy of original student (prevents drift from baseline)\n- **Replay Buffer**: Notebook that stores experiences (question + answer + score)\n- **Trainer**: The tutor that improves the student based on experiences\n\nHere's how these components interact in a conceptual RL step:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import asyncio\nfrom typing import Any, Dict, Optional\n\nimport torch\n\n\ndef conceptual_rl_step():\n \"\"\"\n Conceptual example showing the RL learning flow.\n See apps/grpo/main.py for actual GRPO implementation.\n \"\"\"\n # 1. Get a math problem\n question = \"What is 2+2?\" # dataset.sample()\n\n # 2. Student generates answer\n answer = \"The answer is 4\" # policy.generate(question)\n\n # 3. Teacher grades it\n score = 0.95 # reward_model.evaluate(question, answer)\n\n # 4. Compare to original student\n baseline = 0.85 # reference_model.compute_logprobs(question, answer)\n\n # 5. Store the experience\n experience = {\n \"question\": question,\n \"answer\": answer,\n \"score\": score,\n \"baseline\": baseline,\n }\n # replay_buffer.add(experience)\n\n # 6. When enough experiences collected, improve student\n # trainer.train_step(batch) # Student gets better!\n\n return experience\n\n\nexample_experience = conceptual_rl_step()\nprint(\"Example RL experience:\", example_experience)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## From Concepts to Forge Services\n\nHere's the key insight: **Each RL component becomes a Forge service**.\nThe toy example above maps directly to Forge's distributed architecture:\n\n* Dataset \u2192 DatasetActor\n* Policy \u2192 Policy\n* Reward Model \u2192 RewardActor\n* Reference Model \u2192 ReferenceModel\n* Replay Buffer \u2192 ReplayBuffer\n* Trainer \u2192 RLTrainer\n\nLet's see how the conceptual example translates to actual Forge service calls:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]:\n \"\"\"\n RL step using actual Forge service APIs.\n This shows the same logic as conceptual_rl_step but with real service calls.\n \"\"\"\n # 1. Get a math problem - Using actual DatasetActor API\n sample = await services[\"dataloader\"].sample.call_one()\n prompt, target = sample[\"request\"], sample[\"target\"]\n\n # 2. Student generates answer - Using actual Policy API\n responses = await services[\"policy\"].generate.route(prompt=prompt)\n answer = responses[0].text\n\n # 3. Teacher grades it - Using actual RewardActor API\n score = await services[\"reward_actor\"].evaluate_response.route(\n prompt=prompt, response=answer, target=target\n )\n\n # 4. Compare to baseline - Using actual ReferenceModel API\n # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs\n input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids])\n ref_logprobs = await services[\"ref_model\"].forward.route(\n input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True\n )\n\n # 5. Store experience - Using actual Episode structure from apps/grpo/main.py\n episode = create_episode_from_response(responses[0], score, ref_logprobs, step)\n await services[\"replay_buffer\"].add.call_one(episode)\n\n # 6. Improve student - Using actual trainer pattern\n batch = await services[\"replay_buffer\"].sample.call_one(curr_policy_version=step)\n if batch is not None:\n inputs, targets = batch # GRPO returns (inputs, targets) tuple\n loss = await services[\"trainer\"].train_step.call(inputs, targets)\n\n # 7. Policy synchronization - Using actual weight update pattern\n await services[\"trainer\"].push_weights.call(step + 1)\n await services[\"policy\"].update_weights.fanout(step + 1)\n\n return loss\n\n return None\n\n\ndef create_episode_from_response(response, score, ref_logprobs, step):\n \"\"\"Helper function to create episode from response data\"\"\"\n return {\n \"response\": response,\n \"score\": score,\n \"ref_logprobs\": ref_logprobs,\n \"step\": step,\n }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting Up Forge Services\n\nHere's how to initialize the complete RL system with proper resource allocation.\nEach service can scale independently based on its computational needs:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "async def setup_forge_rl_system():\n \"\"\"\n Complete setup of Forge RL services with proper resource allocation.\n This example uses Qwen 3.1-1.7B model for demonstration.\n \"\"\"\n # Note: In actual Forge environment, imports would be:\n # from forge.actors.policy import Policy\n # from forge.actors.replay_buffer import ReplayBuffer\n # from forge.actors.reference_model import ReferenceModel\n # from forge.actors.trainer import RLTrainer\n # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages\n # from forge.data.rewards import MathReward, ThinkingReward\n\n model = \"Qwen/Qwen3-1.7B\"\n group_size = 1\n\n # Initialize all services with appropriate resource allocation\n services = await asyncio.gather(\n # Dataset actor (CPU intensive for I/O)\n create_dataset_actor(model),\n # Policy service (GPU for inference)\n create_policy_service(model, group_size),\n # Trainer actor (GPU for training)\n create_trainer_actor(model),\n # Replay buffer (CPU for memory management)\n create_replay_buffer_actor(),\n # Advantage computation (CPU)\n create_advantages_actor(),\n # Reference model (GPU for baseline)\n create_reference_model_actor(model),\n # Reward actor (CPU/small GPU for evaluation)\n create_reward_actor(),\n )\n\n service_names = [\n \"dataloader\",\n \"policy\",\n \"trainer\",\n \"replay_buffer\",\n \"compute_advantages\",\n \"ref_model\",\n \"reward_actor\",\n ]\n\n return dict(zip(service_names, services))\n\n\n# Service creation functions (would use actual Forge APIs)\nasync def create_dataset_actor(model):\n \"\"\"DatasetActor for loading training data\"\"\"\n return {\n \"name\": \"DatasetActor\",\n \"config\": {\n \"path\": \"openai/gsm8k\",\n \"revision\": \"main\",\n \"data_split\": \"train\",\n \"streaming\": True,\n \"model\": model,\n },\n \"resources\": \"CPU\",\n \"sample\": lambda: {\n \"call_one\": lambda: {\"request\": \"What is 2+2?\", \"target\": \"4\"}\n },\n }\n\n\nasync def create_policy_service(model, group_size):\n \"\"\"Policy service for text generation\"\"\"\n return {\n \"name\": \"Policy\",\n \"config\": {\n \"engine_config\": {\n \"model\": model,\n \"tensor_parallel_size\": 1,\n \"pipeline_parallel_size\": 1,\n \"enforce_eager\": False,\n },\n \"sampling_config\": {\n \"n\": group_size,\n \"max_tokens\": 16,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n },\n },\n \"resources\": \"GPU\",\n \"generate\": lambda: {\"route\": lambda prompt: [MockResponse()]},\n }\n\n\nasync def create_trainer_actor(model):\n \"\"\"RLTrainer for policy optimization\"\"\"\n return {\n \"name\": \"RLTrainer\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"optimizer\": {\"name\": \"AdamW\", \"lr\": 1e-5},\n \"training\": {\"local_batch_size\": 2, \"seq_len\": 2048},\n },\n \"resources\": \"GPU\",\n \"train_step\": lambda: {\"call\": lambda inputs, targets: 0.5},\n }\n\n\nasync def create_replay_buffer_actor():\n \"\"\"ReplayBuffer for experience storage\"\"\"\n return {\n \"name\": \"ReplayBuffer\",\n \"config\": {\"batch_size\": 2, \"max_policy_age\": 1, \"dp_size\": 1},\n \"resources\": \"CPU\",\n \"add\": lambda: {\"call_one\": lambda episode: None},\n \"sample\": lambda: {\"call_one\": lambda curr_policy_version: ([], [])},\n }\n\n\nasync def create_advantages_actor():\n \"\"\"ComputeAdvantages for advantage estimation\"\"\"\n return {\"name\": \"ComputeAdvantages\", \"resources\": \"CPU\"}\n\n\nasync def create_reference_model_actor(model):\n \"\"\"ReferenceModel for baseline computation\"\"\"\n return {\n \"name\": \"ReferenceModel\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"training\": {\"dtype\": \"bfloat16\"},\n },\n \"resources\": \"GPU\",\n \"forward\": lambda: {\n \"route\": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor(\n [0.1, 0.2]\n )\n },\n }\n\n\nasync def create_reward_actor():\n \"\"\"RewardActor for response evaluation\"\"\"\n return {\n \"name\": \"RewardActor\",\n \"config\": {\"reward_functions\": [\"MathReward\", \"ThinkingReward\"]},\n \"resources\": \"CPU\",\n \"evaluate_response\": lambda: {\"route\": lambda prompt, response, target: 0.95},\n }\n\n\nclass MockResponse:\n \"\"\"Mock response object for demonstration\"\"\"\n\n def __init__(self):\n self.text = \"The answer is 4\"\n self.prompt_ids = torch.tensor([1, 2, 3])\n self.token_ids = torch.tensor([4, 5, 6])\n\n\n# Demonstrate the setup\nprint(\"Setting up Forge RL system...\")\n# services = await setup_forge_rl_system() # Would work in async context\nprint(\"Forge services configured with independent scaling capabilities\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Why Forge Architecture Matters\n\nTraditional ML infrastructure fails for RL because each component has\ndifferent resource needs, scaling patterns, and failure modes:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def show_infrastructure_challenges():\n \"\"\"\n Demonstrate why traditional monolithic RL fails and how Forge solves it.\n \"\"\"\n print(\"=== Infrastructure Challenges ===\\n\")\n\n print(\"Problem 1: Different Resource Needs\")\n resource_requirements = {\n \"Policy (Student AI)\": {\n \"generates\": \"'The answer is 4'\",\n \"needs\": \"Large GPU memory\",\n \"scaling\": \"Multiple replicas for speed\",\n },\n \"Reward Model (Teacher)\": {\n \"scores\": \"answers: 0.95\",\n \"needs\": \"Moderate compute\",\n \"scaling\": \"CPU or small GPU\",\n },\n \"Trainer (Tutor)\": {\n \"improves\": \"student weights\",\n \"needs\": \"Massive GPU compute\",\n \"scaling\": \"Distributed training\",\n },\n \"Dataset (Question Bank)\": {\n \"provides\": \"'What is 2+2?'\",\n \"needs\": \"CPU intensive I/O\",\n \"scaling\": \"High memory bandwidth\",\n },\n }\n\n for component, reqs in resource_requirements.items():\n print(f\"{component}:\")\n for key, value in reqs.items():\n print(f\" {key}: {value}\")\n print()\n\n print(\"Problem 2: Coordination Complexity\")\n print(\"Unlike supervised learning with independent batches,\")\n print(\"RL requires complex coordination between components:\")\n print(\"- Policy waits idle while reward model works\")\n print(\"- Training waits for single episode (batch size = 1)\")\n print(\"- Everything stops if any component fails\")\n print()\n\n print(\"=== Forge Solutions ===\\n\")\n\n print(\"\u2705 Automatic Resource Management\")\n print(\"- Routing to least loaded replica\")\n print(\"- GPU memory management\")\n print(\"- Batch optimization\")\n print(\"- Failure recovery\")\n print(\"- Auto-scaling based on demand\")\n print()\n\n print(\"\u2705 Independent Scaling\")\n print(\"- Policy: num_replicas=8 for high inference demand\")\n print(\"- RewardActor: num_replicas=16 for parallel evaluation\")\n print(\"- Trainer: Multiple actors for distributed training\")\n print()\n\n print(\"\u2705 Fault Tolerance\")\n print(\"- Automatic routing to healthy replicas\")\n print(\"- Background replica respawn\")\n print(\"- Graceful degradation\")\n print(\"- System continues during component failures\")\n\n\nshow_infrastructure_challenges()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Production Scaling Example\n\nHere's how you would scale the system for production workloads:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def demonstrate_production_scaling():\n \"\"\"\n Show how Forge services scale independently for production.\n \"\"\"\n print(\"=== Production Scaling Configuration ===\\n\")\n\n scaling_config = {\n \"Policy Service\": {\n \"replicas\": 8,\n \"reason\": \"High inference demand from multiple training runs\",\n \"resources\": \"GPU-heavy instances\",\n },\n \"RewardActor Service\": {\n \"replicas\": 16,\n \"reason\": \"Parallel evaluation of many responses\",\n \"resources\": \"CPU/small GPU instances\",\n },\n \"Trainer Actor\": {\n \"replicas\": 4,\n \"reason\": \"Distributed training across multiple nodes\",\n \"resources\": \"Large GPU clusters\",\n },\n \"Dataset Actor\": {\n \"replicas\": 2,\n \"reason\": \"I/O intensive data loading\",\n \"resources\": \"High-bandwidth CPU instances\",\n },\n \"ReplayBuffer Actor\": {\n \"replicas\": 1,\n \"reason\": \"Centralized experience storage\",\n \"resources\": \"High-memory instances\",\n },\n }\n\n for service, config in scaling_config.items():\n print(f\"{service}:\")\n print(f\" Replicas: {config['replicas']}\")\n print(f\" Reason: {config['reason']}\")\n print(f\" Resources: {config['resources']}\")\n print()\n\n print(\"Key Benefits:\")\n print(\"- Each service scales based on its bottlenecks\")\n print(\"- Resource utilization is optimized\")\n print(\"- Costs are minimized (no idle GPUs)\")\n print(\"- System maintains performance under load\")\n\n\ndemonstrate_production_scaling()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complete RL Training Loop\n\nHere's a complete example showing multiple RL training steps:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "async def complete_rl_training_example(num_steps: int = 5):\n \"\"\"\n Complete RL training loop using Forge services.\n \"\"\"\n print(f\"=== Running {num_steps} RL Training Steps ===\\n\")\n\n # Setup services (mock for demonstration)\n services = {\n \"dataloader\": await create_dataset_actor(\"Qwen/Qwen3-1.7B\"),\n \"policy\": await create_policy_service(\"Qwen/Qwen3-1.7B\", 1),\n \"trainer\": await create_trainer_actor(\"Qwen/Qwen3-1.7B\"),\n \"replay_buffer\": await create_replay_buffer_actor(),\n \"ref_model\": await create_reference_model_actor(\"Qwen/Qwen3-1.7B\"),\n \"reward_actor\": await create_reward_actor(),\n }\n\n losses = []\n\n for step in range(num_steps):\n print(f\"Step {step + 1}:\")\n\n # Simulate the RL step (would use actual forge_rl_step in practice)\n sample = await services[\"dataloader\"][\"sample\"]()[\"call_one\"]()\n print(f\" Question: {sample['request']}\")\n print(f\" Target: {sample['target']}\")\n\n # Generate response\n responses = await services[\"policy\"][\"generate\"]()[\"route\"](sample[\"request\"])\n print(f\" Generated: {responses[0].text}\")\n\n # Get reward\n score = await services[\"reward_actor\"][\"evaluate_response\"]()[\"route\"](\n sample[\"request\"], responses[0].text, sample[\"target\"]\n )\n print(f\" Reward: {score}\")\n\n # Simulate training (every few steps when buffer has enough data)\n if step >= 2: # Start training after accumulating some experience\n loss = await services[\"trainer\"][\"train_step\"]()[\"call\"]([], [])\n losses.append(loss)\n print(f\" Training Loss: {loss:.4f}\")\n\n print()\n\n print(f\"Training completed! Average loss: {sum(losses)/len(losses):.4f}\")\n return losses\n\n\n# Run the example (would work in async context)\nprint(\"Complete RL training example:\")\nprint(\"(In real usage, run: await complete_rl_training_example(5))\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n\nThis tutorial demonstrated how RL fundamentals map to Forge's distributed\nservice architecture. Key takeaways:\n\n1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.)\n becomes an independent, scalable Forge service\n\n2. **Resource Optimization**: Services scale independently based on their\n computational needs (GPU for inference/training, CPU for data/rewards)\n\n3. **Fault Tolerance**: Individual service failures don't stop the entire\n training pipeline - Forge handles routing and recovery automatically\n\n4. **Simple Interface**: Complex distributed systems are hidden behind\n simple async function calls\n\nThe same RL logic that works conceptually scales to production workloads\nwithout infrastructure code - Forge handles distribution, scaling, and\nfault tolerance automatically.\n\n## Further Reading\n\n* [Forge Architecture Documentation](#)\n* [GRPO Implementation (apps/grpo/main.py)](#)\n* [Forge Service APIs](#)\n* [Production RL Scaling Guide](#)\n\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py new file mode 100644 index 000000000..3e77639a1 --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py @@ -0,0 +1,558 @@ +""" +RL Fundamentals Using Forge Terminology +======================================== + +**Author:** `Your Name `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) + * How RL concepts map to distributed Forge services + * Building scalable RL training loops with fault tolerance + * Resource management and independent scaling patterns + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0+ + * GPU access recommended + * Basic understanding of reinforcement learning + * Familiarity with async/await in Python + +This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. +We'll start with a simple math tutoring example to understand how traditional RL concepts +map to Forge's distributed service model. + +""" + +######################################################################## +# Core RL Components in Forge +# ---------------------------- +# +# Let's start with a simple math tutoring example to understand RL concepts +# with the exact names Forge uses. Think of it as teaching an AI student: +# +# - **Dataset**: Provides questions (like "What is 2+2?") +# - **Policy**: The AI student being trained (generates answers like "The answer is 4") +# - **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) +# - **Reference Model**: Copy of original student (prevents drift from baseline) +# - **Replay Buffer**: Notebook that stores experiences (question + answer + score) +# - **Trainer**: The tutor that improves the student based on experiences +# +# Here's how these components interact in a conceptual RL step: + +import asyncio +from typing import Any, Dict, Optional + +import torch + + +def conceptual_rl_step(): + """ + Conceptual example showing the RL learning flow. + See apps/grpo/main.py for actual GRPO implementation. + """ + # 1. Get a math problem + question = "What is 2+2?" # dataset.sample() + + # 2. Student generates answer + answer = "The answer is 4" # policy.generate(question) + + # 3. Teacher grades it + score = 0.95 # reward_model.evaluate(question, answer) + + # 4. Compare to original student + baseline = 0.85 # reference_model.compute_logprobs(question, answer) + + # 5. Store the experience + experience = { + "question": question, + "answer": answer, + "score": score, + "baseline": baseline, + } + # replay_buffer.add(experience) + + # 6. When enough experiences collected, improve student + # trainer.train_step(batch) # Student gets better! + + return experience + + +example_experience = conceptual_rl_step() +print("Example RL experience:", example_experience) + +######################################################################## +# From Concepts to Forge Services +# -------------------------------- +# +# Here's the key insight: **Each RL component becomes a Forge service**. +# The toy example above maps directly to Forge's distributed architecture: +# +# * Dataset → DatasetActor +# * Policy → Policy +# * Reward Model → RewardActor +# * Reference Model → ReferenceModel +# * Replay Buffer → ReplayBuffer +# * Trainer → RLTrainer +# +# Let's see how the conceptual example translates to actual Forge service calls: + + +async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: + """ + RL step using actual Forge service APIs. + This shows the same logic as conceptual_rl_step but with real service calls. + """ + # 1. Get a math problem - Using actual DatasetActor API + sample = await services["dataloader"].sample.call_one() + prompt, target = sample["request"], sample["target"] + + # 2. Student generates answer - Using actual Policy API + responses = await services["policy"].generate.route(prompt=prompt) + answer = responses[0].text + + # 3. Teacher grades it - Using actual RewardActor API + score = await services["reward_actor"].evaluate_response.route( + prompt=prompt, response=answer, target=target + ) + + # 4. Compare to baseline - Using actual ReferenceModel API + # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs + input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) + ref_logprobs = await services["ref_model"].forward.route( + input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True + ) + + # 5. Store experience - Using actual Episode structure from apps/grpo/main.py + episode = create_episode_from_response(responses[0], score, ref_logprobs, step) + await services["replay_buffer"].add.call_one(episode) + + # 6. Improve student - Using actual trainer pattern + batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) + if batch is not None: + inputs, targets = batch # GRPO returns (inputs, targets) tuple + loss = await services["trainer"].train_step.call(inputs, targets) + + # 7. Policy synchronization - Using actual weight update pattern + await services["trainer"].push_weights.call(step + 1) + await services["policy"].update_weights.fanout(step + 1) + + return loss + + return None + + +def create_episode_from_response(response, score, ref_logprobs, step): + """Helper function to create episode from response data""" + return { + "response": response, + "score": score, + "ref_logprobs": ref_logprobs, + "step": step, + } + + +######################################################################## +# Setting Up Forge Services +# -------------------------- +# +# Here's how to initialize the complete RL system with proper resource allocation. +# Each service can scale independently based on its computational needs: + + +async def setup_forge_rl_system(): + """ + Complete setup of Forge RL services with proper resource allocation. + This example uses Qwen 3.1-1.7B model for demonstration. + """ + # Note: In actual Forge environment, imports would be: + # from forge.actors.policy import Policy + # from forge.actors.replay_buffer import ReplayBuffer + # from forge.actors.reference_model import ReferenceModel + # from forge.actors.trainer import RLTrainer + # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages + # from forge.data.rewards import MathReward, ThinkingReward + + model = "Qwen/Qwen3-1.7B" + group_size = 1 + + # Initialize all services with appropriate resource allocation + services = await asyncio.gather( + # Dataset actor (CPU intensive for I/O) + create_dataset_actor(model), + # Policy service (GPU for inference) + create_policy_service(model, group_size), + # Trainer actor (GPU for training) + create_trainer_actor(model), + # Replay buffer (CPU for memory management) + create_replay_buffer_actor(), + # Advantage computation (CPU) + create_advantages_actor(), + # Reference model (GPU for baseline) + create_reference_model_actor(model), + # Reward actor (CPU/small GPU for evaluation) + create_reward_actor(), + ) + + service_names = [ + "dataloader", + "policy", + "trainer", + "replay_buffer", + "compute_advantages", + "ref_model", + "reward_actor", + ] + + return dict(zip(service_names, services)) + + +# Service creation functions (would use actual Forge APIs) +async def create_dataset_actor(model): + """DatasetActor for loading training data""" + return { + "name": "DatasetActor", + "config": { + "path": "openai/gsm8k", + "revision": "main", + "data_split": "train", + "streaming": True, + "model": model, + }, + "resources": "CPU", + "sample": lambda: { + "call_one": lambda: {"request": "What is 2+2?", "target": "4"} + }, + } + + +async def create_policy_service(model, group_size): + """Policy service for text generation""" + return { + "name": "Policy", + "config": { + "engine_config": { + "model": model, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "enforce_eager": False, + }, + "sampling_config": { + "n": group_size, + "max_tokens": 16, + "temperature": 1.0, + "top_p": 1.0, + }, + }, + "resources": "GPU", + "generate": lambda: {"route": lambda prompt: [MockResponse()]}, + } + + +async def create_trainer_actor(model): + """RLTrainer for policy optimization""" + return { + "name": "RLTrainer", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "optimizer": {"name": "AdamW", "lr": 1e-5}, + "training": {"local_batch_size": 2, "seq_len": 2048}, + }, + "resources": "GPU", + "train_step": lambda: {"call": lambda inputs, targets: 0.5}, + } + + +async def create_replay_buffer_actor(): + """ReplayBuffer for experience storage""" + return { + "name": "ReplayBuffer", + "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, + "resources": "CPU", + "add": lambda: {"call_one": lambda episode: None}, + "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, + } + + +async def create_advantages_actor(): + """ComputeAdvantages for advantage estimation""" + return {"name": "ComputeAdvantages", "resources": "CPU"} + + +async def create_reference_model_actor(model): + """ReferenceModel for baseline computation""" + return { + "name": "ReferenceModel", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "training": {"dtype": "bfloat16"}, + }, + "resources": "GPU", + "forward": lambda: { + "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( + [0.1, 0.2] + ) + }, + } + + +async def create_reward_actor(): + """RewardActor for response evaluation""" + return { + "name": "RewardActor", + "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, + "resources": "CPU", + "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, + } + + +class MockResponse: + """Mock response object for demonstration""" + + def __init__(self): + self.text = "The answer is 4" + self.prompt_ids = torch.tensor([1, 2, 3]) + self.token_ids = torch.tensor([4, 5, 6]) + + +# Demonstrate the setup +print("Setting up Forge RL system...") +# services = await setup_forge_rl_system() # Would work in async context +print("Forge services configured with independent scaling capabilities") + +######################################################################## +# Why Forge Architecture Matters +# ------------------------------- +# +# Traditional ML infrastructure fails for RL because each component has +# different resource needs, scaling patterns, and failure modes: + + +def show_infrastructure_challenges(): + """ + Demonstrate why traditional monolithic RL fails and how Forge solves it. + """ + print("=== Infrastructure Challenges ===\n") + + print("Problem 1: Different Resource Needs") + resource_requirements = { + "Policy (Student AI)": { + "generates": "'The answer is 4'", + "needs": "Large GPU memory", + "scaling": "Multiple replicas for speed", + }, + "Reward Model (Teacher)": { + "scores": "answers: 0.95", + "needs": "Moderate compute", + "scaling": "CPU or small GPU", + }, + "Trainer (Tutor)": { + "improves": "student weights", + "needs": "Massive GPU compute", + "scaling": "Distributed training", + }, + "Dataset (Question Bank)": { + "provides": "'What is 2+2?'", + "needs": "CPU intensive I/O", + "scaling": "High memory bandwidth", + }, + } + + for component, reqs in resource_requirements.items(): + print(f"{component}:") + for key, value in reqs.items(): + print(f" {key}: {value}") + print() + + print("Problem 2: Coordination Complexity") + print("Unlike supervised learning with independent batches,") + print("RL requires complex coordination between components:") + print("- Policy waits idle while reward model works") + print("- Training waits for single episode (batch size = 1)") + print("- Everything stops if any component fails") + print() + + print("=== Forge Solutions ===\n") + + print("✅ Automatic Resource Management") + print("- Routing to least loaded replica") + print("- GPU memory management") + print("- Batch optimization") + print("- Failure recovery") + print("- Auto-scaling based on demand") + print() + + print("✅ Independent Scaling") + print("- Policy: num_replicas=8 for high inference demand") + print("- RewardActor: num_replicas=16 for parallel evaluation") + print("- Trainer: Multiple actors for distributed training") + print() + + print("✅ Fault Tolerance") + print("- Automatic routing to healthy replicas") + print("- Background replica respawn") + print("- Graceful degradation") + print("- System continues during component failures") + + +show_infrastructure_challenges() + +######################################################################## +# Production Scaling Example +# --------------------------- +# +# Here's how you would scale the system for production workloads: + + +def demonstrate_production_scaling(): + """ + Show how Forge services scale independently for production. + """ + print("=== Production Scaling Configuration ===\n") + + scaling_config = { + "Policy Service": { + "replicas": 8, + "reason": "High inference demand from multiple training runs", + "resources": "GPU-heavy instances", + }, + "RewardActor Service": { + "replicas": 16, + "reason": "Parallel evaluation of many responses", + "resources": "CPU/small GPU instances", + }, + "Trainer Actor": { + "replicas": 4, + "reason": "Distributed training across multiple nodes", + "resources": "Large GPU clusters", + }, + "Dataset Actor": { + "replicas": 2, + "reason": "I/O intensive data loading", + "resources": "High-bandwidth CPU instances", + }, + "ReplayBuffer Actor": { + "replicas": 1, + "reason": "Centralized experience storage", + "resources": "High-memory instances", + }, + } + + for service, config in scaling_config.items(): + print(f"{service}:") + print(f" Replicas: {config['replicas']}") + print(f" Reason: {config['reason']}") + print(f" Resources: {config['resources']}") + print() + + print("Key Benefits:") + print("- Each service scales based on its bottlenecks") + print("- Resource utilization is optimized") + print("- Costs are minimized (no idle GPUs)") + print("- System maintains performance under load") + + +demonstrate_production_scaling() + +######################################################################## +# Complete RL Training Loop +# -------------------------- +# +# Here's a complete example showing multiple RL training steps: + + +async def complete_rl_training_example(num_steps: int = 5): + """ + Complete RL training loop using Forge services. + """ + print(f"=== Running {num_steps} RL Training Steps ===\n") + + # Setup services (mock for demonstration) + services = { + "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), + "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), + "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), + "replay_buffer": await create_replay_buffer_actor(), + "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), + "reward_actor": await create_reward_actor(), + } + + losses = [] + + for step in range(num_steps): + print(f"Step {step + 1}:") + + # Simulate the RL step (would use actual forge_rl_step in practice) + sample = await services["dataloader"]["sample"]()["call_one"]() + print(f" Question: {sample['request']}") + print(f" Target: {sample['target']}") + + # Generate response + responses = await services["policy"]["generate"]()["route"](sample["request"]) + print(f" Generated: {responses[0].text}") + + # Get reward + score = await services["reward_actor"]["evaluate_response"]()["route"]( + sample["request"], responses[0].text, sample["target"] + ) + print(f" Reward: {score}") + + # Simulate training (every few steps when buffer has enough data) + if step >= 2: # Start training after accumulating some experience + loss = await services["trainer"]["train_step"]()["call"]([], []) + losses.append(loss) + print(f" Training Loss: {loss:.4f}") + + print() + + print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") + return losses + + +# Run the example (would work in async context) +print("Complete RL training example:") +print("(In real usage, run: await complete_rl_training_example(5))") + +######################################################################## +# Conclusion +# ---------- +# +# This tutorial demonstrated how RL fundamentals map to Forge's distributed +# service architecture. Key takeaways: +# +# 1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) +# becomes an independent, scalable Forge service +# +# 2. **Resource Optimization**: Services scale independently based on their +# computational needs (GPU for inference/training, CPU for data/rewards) +# +# 3. **Fault Tolerance**: Individual service failures don't stop the entire +# training pipeline - Forge handles routing and recovery automatically +# +# 4. **Simple Interface**: Complex distributed systems are hidden behind +# simple async function calls +# +# The same RL logic that works conceptually scales to production workloads +# without infrastructure code - Forge handles distribution, scaling, and +# fault tolerance automatically. +# +# Further Reading +# --------------- +# +# * `Forge Architecture Documentation <#>`_ +# * `GRPO Implementation (apps/grpo/main.py) <#>`_ +# * `Forge Service APIs <#>`_ +# * `Production RL Scaling Guide <#>`_ + diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 new file mode 100644 index 000000000..d93f4a3ab --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 @@ -0,0 +1 @@ +d1dbe1df9283c920a3039b06b0d5ce4d \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst b/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst new file mode 100644 index 000000000..95ad7c0ba --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst @@ -0,0 +1,788 @@ + +.. DO NOT EDIT. +.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. +.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: +.. "tutorials/rl_fundamentals_forge_tutorial.py" +.. LINE NUMBERS ARE GIVEN BELOW. + +.. only:: html + + .. note:: + :class: sphx-glr-download-link-note + + :ref:`Go to the end ` + to download the full example code. + +.. rst-class:: sphx-glr-example-title + +.. _sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py: + + +RL Fundamentals Using Forge Terminology +======================================== + +**Author:** `Your Name `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) + * How RL concepts map to distributed Forge services + * Building scalable RL training loops with fault tolerance + * Resource management and independent scaling patterns + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0+ + * GPU access recommended + * Basic understanding of reinforcement learning + * Familiarity with async/await in Python + +This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. +We'll start with a simple math tutoring example to understand how traditional RL concepts +map to Forge's distributed service model. + +.. GENERATED FROM PYTHON SOURCE LINES 32-46 + +Core RL Components in Forge +---------------------------- + +Let's start with a simple math tutoring example to understand RL concepts +with the exact names Forge uses. Think of it as teaching an AI student: + +- **Dataset**: Provides questions (like "What is 2+2?") +- **Policy**: The AI student being trained (generates answers like "The answer is 4") +- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) +- **Reference Model**: Copy of original student (prevents drift from baseline) +- **Replay Buffer**: Notebook that stores experiences (question + answer + score) +- **Trainer**: The tutor that improves the student based on experiences + +Here's how these components interact in a conceptual RL step: + +.. GENERATED FROM PYTHON SOURCE LINES 46-88 + +.. code-block:: Python + + + import asyncio + from typing import Any, Dict, Optional + + import torch + + + def conceptual_rl_step(): + """ + Conceptual example showing the RL learning flow. + See apps/grpo/main.py for actual GRPO implementation. + """ + # 1. Get a math problem + question = "What is 2+2?" # dataset.sample() + + # 2. Student generates answer + answer = "The answer is 4" # policy.generate(question) + + # 3. Teacher grades it + score = 0.95 # reward_model.evaluate(question, answer) + + # 4. Compare to original student + baseline = 0.85 # reference_model.compute_logprobs(question, answer) + + # 5. Store the experience + experience = { + "question": question, + "answer": answer, + "score": score, + "baseline": baseline, + } + # replay_buffer.add(experience) + + # 6. When enough experiences collected, improve student + # trainer.train_step(batch) # Student gets better! + + return experience + + + example_experience = conceptual_rl_step() + print("Example RL experience:", example_experience) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Example RL experience: {'question': 'What is 2+2?', 'answer': 'The answer is 4', 'score': 0.95, 'baseline': 0.85} + + + + +.. GENERATED FROM PYTHON SOURCE LINES 89-103 + +From Concepts to Forge Services +-------------------------------- + +Here's the key insight: **Each RL component becomes a Forge service**. +The toy example above maps directly to Forge's distributed architecture: + +* Dataset → DatasetActor +* Policy → Policy +* Reward Model → RewardActor +* Reference Model → ReferenceModel +* Replay Buffer → ReplayBuffer +* Trainer → RLTrainer + +Let's see how the conceptual example translates to actual Forge service calls: + +.. GENERATED FROM PYTHON SOURCE LINES 103-159 + +.. code-block:: Python + + + + async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: + """ + RL step using actual Forge service APIs. + This shows the same logic as conceptual_rl_step but with real service calls. + """ + # 1. Get a math problem - Using actual DatasetActor API + sample = await services["dataloader"].sample.call_one() + prompt, target = sample["request"], sample["target"] + + # 2. Student generates answer - Using actual Policy API + responses = await services["policy"].generate.route(prompt=prompt) + answer = responses[0].text + + # 3. Teacher grades it - Using actual RewardActor API + score = await services["reward_actor"].evaluate_response.route( + prompt=prompt, response=answer, target=target + ) + + # 4. Compare to baseline - Using actual ReferenceModel API + # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs + input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) + ref_logprobs = await services["ref_model"].forward.route( + input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True + ) + + # 5. Store experience - Using actual Episode structure from apps/grpo/main.py + episode = create_episode_from_response(responses[0], score, ref_logprobs, step) + await services["replay_buffer"].add.call_one(episode) + + # 6. Improve student - Using actual trainer pattern + batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) + if batch is not None: + inputs, targets = batch # GRPO returns (inputs, targets) tuple + loss = await services["trainer"].train_step.call(inputs, targets) + + # 7. Policy synchronization - Using actual weight update pattern + await services["trainer"].push_weights.call(step + 1) + await services["policy"].update_weights.fanout(step + 1) + + return loss + + return None + + + def create_episode_from_response(response, score, ref_logprobs, step): + """Helper function to create episode from response data""" + return { + "response": response, + "score": score, + "ref_logprobs": ref_logprobs, + "step": step, + } + + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 160-165 + +Setting Up Forge Services +-------------------------- + +Here's how to initialize the complete RL system with proper resource allocation. +Each service can scale independently based on its computational needs: + +.. GENERATED FROM PYTHON SOURCE LINES 165-335 + +.. code-block:: Python + + + + async def setup_forge_rl_system(): + """ + Complete setup of Forge RL services with proper resource allocation. + This example uses Qwen 3.1-1.7B model for demonstration. + """ + # Note: In actual Forge environment, imports would be: + # from forge.actors.policy import Policy + # from forge.actors.replay_buffer import ReplayBuffer + # from forge.actors.reference_model import ReferenceModel + # from forge.actors.trainer import RLTrainer + # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages + # from forge.data.rewards import MathReward, ThinkingReward + + model = "Qwen/Qwen3-1.7B" + group_size = 1 + + # Initialize all services with appropriate resource allocation + services = await asyncio.gather( + # Dataset actor (CPU intensive for I/O) + create_dataset_actor(model), + # Policy service (GPU for inference) + create_policy_service(model, group_size), + # Trainer actor (GPU for training) + create_trainer_actor(model), + # Replay buffer (CPU for memory management) + create_replay_buffer_actor(), + # Advantage computation (CPU) + create_advantages_actor(), + # Reference model (GPU for baseline) + create_reference_model_actor(model), + # Reward actor (CPU/small GPU for evaluation) + create_reward_actor(), + ) + + service_names = [ + "dataloader", + "policy", + "trainer", + "replay_buffer", + "compute_advantages", + "ref_model", + "reward_actor", + ] + + return dict(zip(service_names, services)) + + + # Service creation functions (would use actual Forge APIs) + async def create_dataset_actor(model): + """DatasetActor for loading training data""" + return { + "name": "DatasetActor", + "config": { + "path": "openai/gsm8k", + "revision": "main", + "data_split": "train", + "streaming": True, + "model": model, + }, + "resources": "CPU", + "sample": lambda: { + "call_one": lambda: {"request": "What is 2+2?", "target": "4"} + }, + } + + + async def create_policy_service(model, group_size): + """Policy service for text generation""" + return { + "name": "Policy", + "config": { + "engine_config": { + "model": model, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "enforce_eager": False, + }, + "sampling_config": { + "n": group_size, + "max_tokens": 16, + "temperature": 1.0, + "top_p": 1.0, + }, + }, + "resources": "GPU", + "generate": lambda: {"route": lambda prompt: [MockResponse()]}, + } + + + async def create_trainer_actor(model): + """RLTrainer for policy optimization""" + return { + "name": "RLTrainer", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "optimizer": {"name": "AdamW", "lr": 1e-5}, + "training": {"local_batch_size": 2, "seq_len": 2048}, + }, + "resources": "GPU", + "train_step": lambda: {"call": lambda inputs, targets: 0.5}, + } + + + async def create_replay_buffer_actor(): + """ReplayBuffer for experience storage""" + return { + "name": "ReplayBuffer", + "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, + "resources": "CPU", + "add": lambda: {"call_one": lambda episode: None}, + "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, + } + + + async def create_advantages_actor(): + """ComputeAdvantages for advantage estimation""" + return {"name": "ComputeAdvantages", "resources": "CPU"} + + + async def create_reference_model_actor(model): + """ReferenceModel for baseline computation""" + return { + "name": "ReferenceModel", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "training": {"dtype": "bfloat16"}, + }, + "resources": "GPU", + "forward": lambda: { + "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( + [0.1, 0.2] + ) + }, + } + + + async def create_reward_actor(): + """RewardActor for response evaluation""" + return { + "name": "RewardActor", + "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, + "resources": "CPU", + "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, + } + + + class MockResponse: + """Mock response object for demonstration""" + + def __init__(self): + self.text = "The answer is 4" + self.prompt_ids = torch.tensor([1, 2, 3]) + self.token_ids = torch.tensor([4, 5, 6]) + + + # Demonstrate the setup + print("Setting up Forge RL system...") + # services = await setup_forge_rl_system() # Would work in async context + print("Forge services configured with independent scaling capabilities") + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Setting up Forge RL system... + Forge services configured with independent scaling capabilities + + + + +.. GENERATED FROM PYTHON SOURCE LINES 336-341 + +Why Forge Architecture Matters +------------------------------- + +Traditional ML infrastructure fails for RL because each component has +different resource needs, scaling patterns, and failure modes: + +.. GENERATED FROM PYTHON SOURCE LINES 341-412 + +.. code-block:: Python + + + + def show_infrastructure_challenges(): + """ + Demonstrate why traditional monolithic RL fails and how Forge solves it. + """ + print("=== Infrastructure Challenges ===\n") + + print("Problem 1: Different Resource Needs") + resource_requirements = { + "Policy (Student AI)": { + "generates": "'The answer is 4'", + "needs": "Large GPU memory", + "scaling": "Multiple replicas for speed", + }, + "Reward Model (Teacher)": { + "scores": "answers: 0.95", + "needs": "Moderate compute", + "scaling": "CPU or small GPU", + }, + "Trainer (Tutor)": { + "improves": "student weights", + "needs": "Massive GPU compute", + "scaling": "Distributed training", + }, + "Dataset (Question Bank)": { + "provides": "'What is 2+2?'", + "needs": "CPU intensive I/O", + "scaling": "High memory bandwidth", + }, + } + + for component, reqs in resource_requirements.items(): + print(f"{component}:") + for key, value in reqs.items(): + print(f" {key}: {value}") + print() + + print("Problem 2: Coordination Complexity") + print("Unlike supervised learning with independent batches,") + print("RL requires complex coordination between components:") + print("- Policy waits idle while reward model works") + print("- Training waits for single episode (batch size = 1)") + print("- Everything stops if any component fails") + print() + + print("=== Forge Solutions ===\n") + + print("✅ Automatic Resource Management") + print("- Routing to least loaded replica") + print("- GPU memory management") + print("- Batch optimization") + print("- Failure recovery") + print("- Auto-scaling based on demand") + print() + + print("✅ Independent Scaling") + print("- Policy: num_replicas=8 for high inference demand") + print("- RewardActor: num_replicas=16 for parallel evaluation") + print("- Trainer: Multiple actors for distributed training") + print() + + print("✅ Fault Tolerance") + print("- Automatic routing to healthy replicas") + print("- Background replica respawn") + print("- Graceful degradation") + print("- System continues during component failures") + + + show_infrastructure_challenges() + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + === Infrastructure Challenges === + + Problem 1: Different Resource Needs + Policy (Student AI): + generates: 'The answer is 4' + needs: Large GPU memory + scaling: Multiple replicas for speed + + Reward Model (Teacher): + scores: answers: 0.95 + needs: Moderate compute + scaling: CPU or small GPU + + Trainer (Tutor): + improves: student weights + needs: Massive GPU compute + scaling: Distributed training + + Dataset (Question Bank): + provides: 'What is 2+2?' + needs: CPU intensive I/O + scaling: High memory bandwidth + + Problem 2: Coordination Complexity + Unlike supervised learning with independent batches, + RL requires complex coordination between components: + - Policy waits idle while reward model works + - Training waits for single episode (batch size = 1) + - Everything stops if any component fails + + === Forge Solutions === + + ✅ Automatic Resource Management + - Routing to least loaded replica + - GPU memory management + - Batch optimization + - Failure recovery + - Auto-scaling based on demand + + ✅ Independent Scaling + - Policy: num_replicas=8 for high inference demand + - RewardActor: num_replicas=16 for parallel evaluation + - Trainer: Multiple actors for distributed training + + ✅ Fault Tolerance + - Automatic routing to healthy replicas + - Background replica respawn + - Graceful degradation + - System continues during component failures + + + + +.. GENERATED FROM PYTHON SOURCE LINES 413-417 + +Production Scaling Example +--------------------------- + +Here's how you would scale the system for production workloads: + +.. GENERATED FROM PYTHON SOURCE LINES 417-469 + +.. code-block:: Python + + + + def demonstrate_production_scaling(): + """ + Show how Forge services scale independently for production. + """ + print("=== Production Scaling Configuration ===\n") + + scaling_config = { + "Policy Service": { + "replicas": 8, + "reason": "High inference demand from multiple training runs", + "resources": "GPU-heavy instances", + }, + "RewardActor Service": { + "replicas": 16, + "reason": "Parallel evaluation of many responses", + "resources": "CPU/small GPU instances", + }, + "Trainer Actor": { + "replicas": 4, + "reason": "Distributed training across multiple nodes", + "resources": "Large GPU clusters", + }, + "Dataset Actor": { + "replicas": 2, + "reason": "I/O intensive data loading", + "resources": "High-bandwidth CPU instances", + }, + "ReplayBuffer Actor": { + "replicas": 1, + "reason": "Centralized experience storage", + "resources": "High-memory instances", + }, + } + + for service, config in scaling_config.items(): + print(f"{service}:") + print(f" Replicas: {config['replicas']}") + print(f" Reason: {config['reason']}") + print(f" Resources: {config['resources']}") + print() + + print("Key Benefits:") + print("- Each service scales based on its bottlenecks") + print("- Resource utilization is optimized") + print("- Costs are minimized (no idle GPUs)") + print("- System maintains performance under load") + + + demonstrate_production_scaling() + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + === Production Scaling Configuration === + + Policy Service: + Replicas: 8 + Reason: High inference demand from multiple training runs + Resources: GPU-heavy instances + + RewardActor Service: + Replicas: 16 + Reason: Parallel evaluation of many responses + Resources: CPU/small GPU instances + + Trainer Actor: + Replicas: 4 + Reason: Distributed training across multiple nodes + Resources: Large GPU clusters + + Dataset Actor: + Replicas: 2 + Reason: I/O intensive data loading + Resources: High-bandwidth CPU instances + + ReplayBuffer Actor: + Replicas: 1 + Reason: Centralized experience storage + Resources: High-memory instances + + Key Benefits: + - Each service scales based on its bottlenecks + - Resource utilization is optimized + - Costs are minimized (no idle GPUs) + - System maintains performance under load + + + + +.. GENERATED FROM PYTHON SOURCE LINES 470-474 + +Complete RL Training Loop +-------------------------- + +Here's a complete example showing multiple RL training steps: + +.. GENERATED FROM PYTHON SOURCE LINES 474-528 + +.. code-block:: Python + + + + async def complete_rl_training_example(num_steps: int = 5): + """ + Complete RL training loop using Forge services. + """ + print(f"=== Running {num_steps} RL Training Steps ===\n") + + # Setup services (mock for demonstration) + services = { + "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), + "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), + "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), + "replay_buffer": await create_replay_buffer_actor(), + "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), + "reward_actor": await create_reward_actor(), + } + + losses = [] + + for step in range(num_steps): + print(f"Step {step + 1}:") + + # Simulate the RL step (would use actual forge_rl_step in practice) + sample = await services["dataloader"]["sample"]()["call_one"]() + print(f" Question: {sample['request']}") + print(f" Target: {sample['target']}") + + # Generate response + responses = await services["policy"]["generate"]()["route"](sample["request"]) + print(f" Generated: {responses[0].text}") + + # Get reward + score = await services["reward_actor"]["evaluate_response"]()["route"]( + sample["request"], responses[0].text, sample["target"] + ) + print(f" Reward: {score}") + + # Simulate training (every few steps when buffer has enough data) + if step >= 2: # Start training after accumulating some experience + loss = await services["trainer"]["train_step"]()["call"]([], []) + losses.append(loss) + print(f" Training Loss: {loss:.4f}") + + print() + + print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") + return losses + + + # Run the example (would work in async context) + print("Complete RL training example:") + print("(In real usage, run: await complete_rl_training_example(5))") + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Complete RL training example: + (In real usage, run: await complete_rl_training_example(5)) + + + + +.. GENERATED FROM PYTHON SOURCE LINES 529-558 + +Conclusion +---------- + +This tutorial demonstrated how RL fundamentals map to Forge's distributed +service architecture. Key takeaways: + +1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) + becomes an independent, scalable Forge service + +2. **Resource Optimization**: Services scale independently based on their + computational needs (GPU for inference/training, CPU for data/rewards) + +3. **Fault Tolerance**: Individual service failures don't stop the entire + training pipeline - Forge handles routing and recovery automatically + +4. **Simple Interface**: Complex distributed systems are hidden behind + simple async function calls + +The same RL logic that works conceptually scales to production workloads +without infrastructure code - Forge handles distribution, scaling, and +fault tolerance automatically. + +Further Reading +--------------- + +* `Forge Architecture Documentation <#>`_ +* `GRPO Implementation (apps/grpo/main.py) <#>`_ +* `Forge Service APIs <#>`_ +* `Production RL Scaling Guide <#>`_ + + +.. rst-class:: sphx-glr-timing + + **Total running time of the script:** (0 minutes 0.006 seconds) + + +.. _sphx_glr_download_tutorials_rl_fundamentals_forge_tutorial.py: + +.. only:: html + + .. container:: sphx-glr-footer sphx-glr-footer-example + + .. container:: sphx-glr-download sphx-glr-download-jupyter + + :download:`Download Jupyter notebook: rl_fundamentals_forge_tutorial.ipynb ` + + .. container:: sphx-glr-download sphx-glr-download-python + + :download:`Download Python source code: rl_fundamentals_forge_tutorial.py ` + + .. container:: sphx-glr-download sphx-glr-download-zip + + :download:`Download zipped: rl_fundamentals_forge_tutorial.zip ` diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip b/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip new file mode 100644 index 0000000000000000000000000000000000000000..c77b9084e4c0471638195cc647797a00896fc0af GIT binary patch literal 40960 zcmdU2&2Ah?a+ZG&ycyUBs4d!R<;m;HKC zPv@gz+?`$JdA@P@b@nW1lzmf;Cg<5RHqBm^^UKj>I-Z_iZ9HD{&&I~~_LD_@F`e&k zZ)e~A<#aL6ehn6~pI_AVtlGbS|9n(mEKa+F>E-<^JY7`fe1ZnQ_^!XP(d}mE^U-jB zKYO^bk>QW|V0ToPm%D>v&PV&xK|LBwC;Q)BPUm$o-#wk;!JqGymw*0U_U%PcXP}$C z8I8x;xGd%qAp~0cgK<$+N+lz$-PydHm)|c&706fmLbTh?o=)dw2FZZO*>nQ&RoQ4_ zgwpw`$$qvgmAeOzD4$JXQR@Q^=mVM=R_RHxTr8F6oGtjv#W?4PW zhNG&Uk4_hLIW%-3hS#G3*x<17XFL0RF&Ym!8`Yo~7pG%ksh$_32|pZ9r?U#9tS_>& zVll2kbzII1yv9>FEGtNOP~xqV;=H6HWW{8N!3@h;iT~ku5+}V`QP-FWBZk#-AA|ed z#bi1kLVi~A2azy!5I2zd!PU#@d~lJye%O7`eee@Mh_4R5$%+9aQ)TlKGlEeJ%b|b% zc~Ol98FaXuS2ae*!A#H4b~J&;4|LR`2zcVVJu5Cp<54jm)mJLTqPm(4?iX*0QO(&t zxPlT-Ha1>fjH=8onyfC1!9`hdO2f64E&Q_isLIN>#Q+oHmybxj!0@1GgL*M9yBpt@ zA3;IE@w_(g&#KYo46}S$U{3Xo7z${~r%)O{#_R%W1yvZ1>d|ykj6)UMFe)Z%7|evK zW|S1HfRAhHt*=q#->|1PumLzPj^@R+1Nybuge+?-&Q8FV0D>+maL5WqBN61 z6H}R`29r~DGfeE{6*UF(U0{kiv(%kpl0A72CKg;C`@+|5w!LkZ{r2{Lb}*m59t}b6 z`$buC0;;St9=$5FT&xfD<>5~r{zbmUA)}|-33hx5c0BFu6e@boX&g;p20G{E1Pd6J z2rN`@pv{>QWiS0C1L;SOZfJKbb=A*#R?`fYuZ!^_AP9c3)}im6^U-U(rW(M`nYTRX z{_GRW@L_p|bqz_XO(X56)7cee1=n#tqQV*ZWF6Sh*J2@u^U+zIoz170*(nSpj7sSV z&c?+R4B8n8liaVTb$L3SzEVCbcqH&qzMYlx5#I}WTG3`dv19*<@@r|n6tM|43P{$A zlEW&(1c3-=-||V8hS_uyzD4Html!IPP8Jkab{QLetYI!?*n@{hGn|bW}onX0z)4`Fu9Le+gZK=g(jnVN`35 z<*UPk-;md_SjvJNYES#x&1|omeN|$RYMLNjIHOCU?xyK+L;168GaIU>>{epII$P>t zH?xP`>`0ea*3@|+YLp!$o2pN$GjY3J+tX=^BmXhhwwUQTf{B9bbW{sn(f90eMqMYR zxj62=8biBS9K9W*c#iI)E@t3zRvbB|d4pAadC6zyC1x?1_b{e_+wUW)q9Ci~ z@B4&eVkAO6_KQPa~1Nk@j+ztqj?2r2N zy-=kLxwQMIvb4L!aM5?MJbe&Q!dUOfjnWZ5qZKvF9d>ML9M_SJ5m)#&|)*>PvZDJKen2Ez+PW%c;VaoGqt!l*~R z>2_v(Y4-1b`*-%|Cj+QD-n4DPYI&xfQ$7uyojlVIe7AW zNfYB)^x@a$w=h0|MK$gs&>klnDC(pdi`&J-86&G3DjR+w{e^QV$FTQBeG~r|8$C(m4&bi}5*p^e6;+T$e zZIhp@;h>s)tBN{~5yxQ1i1;$Wv^p*@Y+YA(K7}Vh1Vv6C>p#l8^MH=t@q?3YUB0a? zizz49qnONwct_xZTZH!n(;&9bX?0*^!x3IxMgf4uO&UlJ+NZT%&VS;J-Nx~&z zyhTs#2uW>bKkZtl%U#UHd^#EZPWAy!DZVKgKV*v;>~9&#l7yNj#WOQoR2O~iTq!%0 z7q)hLDzj8?XD#(@uKQVmAhPaw0!}6-ndG!9`={z7`s&_fsl;E zwi2#h7ei)OhinDy8j!t^-B1}ugRtE;Lzr!U41(}w!Fjlrq-U_SoK$j3ijh73v7!5& z#;QjMySWAZW_FV(HH=Rsn3`g3Ae)5*#qV^-!;R9ogq|^uxdQu_8gRf2Q}U3L<~HH> zr!dyCZQaZyu=U}{1a_xDPA$lIB2=Z%kz~LK+Xmf`2?0bhld>E(LNg?L7c-MRXMiTW zwD;zQTWQTyiH3Y+-bC%!v(I%PftcOiGgFoQ_6@d9A9we5_qsp*TvK7LP6+4l`y{nGaubK`os$US%+{aKfZkS-G;wrLo9 z+H}4IqC2wTM~wDl_`1L$%DG9;N6a%8?P~ahutU>9FN*p?qpuxK=;Rd^i2kmYN!&l| zch0%J&;NZav#WWYbHs(Se#M3GINP%k-*dk_pz;l^!oaXBaIS;(l2{ocG^dBwZkeg7*~=zNa|SeMC4y;LeTPTsf$`b{4L@}UzW0~DvON{la?w2 za(jtYH@`wqX-`yIgrfyu$VZ$yZ3=CVyrD&_aEg~AbZCQMV-=i8ZBX5!2bZC#U5!9j z3|_)%KmYfuOKP2!s@WgF=q6slt}(kNQZp0T*JS+T?AWPXxTEtrlbawQ_{7Xw9()wZ zJA7bMZC;4sQ+iJg!y{UrdDGbJ3XRhqB9Ytq-DuW{#)6Q+82K#>6w3z^qu6F9dL z$8-~Nz-^76D0zd<-o7fofA-%0h|Ru62^HS~GZwrd*SH=J#>M67uxJg^CKSRJE_(oS z(P7UHh9H>+zUWb|=}~fNxOpFCd(7NkYv0^NtIxGsCR5F9fX$y%%;u1w9bDQ;RvMuM zmQ8Ot!68rCU-EdB?D>p;aypACwn@btW}`&Z1aAh>?D@IThE%4}Oiqi|YFz4}Zoh;x zo`c{sR5mp9BA9<+Mdlewv~oODQ}0+3Tr!KG83YdF(?knIs0d689`CYA_d%-7bk=7y z-g+7^9sC&UFN|+Edp3;(a!!MDNh!HcOcrcEJAN@8ygD>Vn$FhADy>X8MyH5}HmnjI zR0YnaIM=!~d&cYa(2)oSvqaZ&yT)S!B=c(Z`g?{?0WGIkXXD~EvRPP=jGR*~F3$QW z2`TFex+fOxEWbEIjmkTD?fXWTU}A&;Gi=erce!Qf2`X;B#Th|9=BxM0-A|hRM8Im* z-sRl9jQf%Wu!{sqoxDQAYFtt)A3k{WSwJE9aiw<1OC$l{v;h__r%`hsAoN0(%%>yv zaWrPRO&YAo)mj(tbt4G!MLmLi6VG_DWY^G+fJh+!+$%`l5*F8{v88+mqz(xVHH7AG zOJ&yZ2uLzH*;rvQrR+Ymn|q#MPs)&N+nI={4m=+50pPv@7^Ao|lG4!2X2ct5erjNYn5RF3sG`EVr`C`z9-{ zZryJct*t$N(8Zd2(0zCk$=`1dR%v&5_7LP%bi*tganTuQq@1nTx|W1WyO%rN4v3`n z1p<|j-v#K9-f`|X@LUV=7CyJ$Q#7o#PQ2I8Tc;@D%%oj4C9lJ@Nn#r1pM_Rg${10a z;~lzNY5dR;$fl=%g`-)KU6d>0LK8{x;IZFl-mTxqhGu+b3QjmM{wgO($AwKiGS?=_xo)>BRZ@0ShPT+{bTXC4bkw(!1$#4{zk*lPdLEv8%9)!f2Y4n` zv5u&EBt&~d&Flp^M#(yIA2ujv#VP6x>rq*OrMs*V`1az;g@qY37gZ1ygtwyF z&Q%D!3~C5pe9ctMya>*KiqTm82}Fwmfr6V>mez>bh6hCb=taSI4M)t9@OQf&G?R*r zF(M91!q%6vtY(Ok)zKJ|ldYSiTd9gXeug%NEc6E#+&@f^c?&Fqs9&fXZy*bpQ&$d& z#3+)K^#zWv!8u>XqPn3R+nZUOj#(l$s^imYGt($`M(#9ZBzx)@$?&u{;cQZ$O^*XD ziputQWN&40XmixRW>y(J*ue^jXOasPPcRwFb>Ji1-0b!`w!G!Z^DS5o-+*Lt8~)4I z_M^rM{U|VY(pvJ`Ad-L0U8>}9G?UeAXy%JeU!bgb#EMjgeWL*i0hj=*W*AJ+D|Ak3 zZMAGL>zFF61n+X9A&je4sno~?RAZ)J;B1nmktSRyn+*e|nX4q1HeD4o4LOD4V~dNt z-b>cQCs;JqNW!EoA~Z)>0m%lN*i0}($}E{lJ~i70oPH71fV)`0dykM?lb^{tzcm%7 z+2_UNRf21nVku4K^nDa2D3&WkoZLlXcLmG8#QC4ifSzJOy%`O$ZEEq1P?3V6wonXW zDCpR}$AX)r7eJR4{%(!4w=S%+{GB85e%~x1OB_W_I~j{ydB7v|+hjsgS@sTH-eW0A zr}qIleOr1>KVK4nKmRBwG(R93^xM3R;_V(ZZ{%Af%{Z%Dux+WpLV6FF2+UlQI;a3(M$&>r?Px$N@NUUq2?+3N)NI%dKzP@`5OM6fbw zcKW;+ykhbX5rS!gf2&#XhF;zq5J==Jl!KRNi!lVo+G5>TH`*NOVKsxt(FFE78!k{z zidfK^LL2>X0d8zu7xmm}(1E*mV|_o{fl4ZLUN;guk=_m|Yv$9Q9y8 z*Yo^vDbEPbV6*KxJP**si@~_86fQMv*M)1D27Id}d>CY5*AR2#SSL`8r`X-#2nYeg zFsPwtbRg;A)=&dg7PgHKUZD{|uxYHr4oof2Eg2NX%eNgKs;5SjY5GPnrhS;f6RP(_ z=%}=hKeCJaqmy@BbVZ$j%V;wL-$EC`v3R zZcBtUP^Z(n#xA=Yw1YwyTEUxQ8Z`n&638BY#9<0tt|@wdtlKf{*8eBcV2TzV8W7jJo;_qbDUse!(hpoz_)8~+*?Z&y0Y z&{y)-m?;diHC^=gq~nYXf_`V;S*6z~KeGgQIl#8V#S^_##mG+k&@jL|ebTx!W?mn@ zg?ER;E35Wgxly`r)t@o>;bJ1!j^DXAzGrI1N#%%)$DWf_5JYBH3yYH6(i{?NRtEc2;6*`c$aO2p5F#A*EH z5=qJ~)AqlVQjnm%)`n~es)3`0+MPCAB3cwu2O z53=umQ1;bBaOqxVZ<)}eIabIf^SV?F_u+c?NRAUnAQ7vMo}anshE9t!B$S(%cG#7F zQd6fvE+>N54;sVrI%lh;I=IV>;=Z%;jnqq{HkQ{GO$i_}-S!3}&60l|k{qJezj&M> zRYYaRbmRWNrCOXJE<|iOScs0(_*b}V7sNx67_O`B$9*MI*0X)5zfb{lklZ4M+wQ0j z1#Cixq&?phnpX&C%D+YRIb7sR)>UEukfpt^{B7}khiu%fgdeq_W8rT9a&T+|Y| z+6w>7mCCxE_kF7^sQNaQ$z105)$n22P&T-Bx4qp+L6IjU<3>ol^FFjowvgtsv*jx-$-ACr|^s5E%MohDfg9!EM-p~Kz zzyIW$d-wSFPp?OP8qKaIr#wl_2PN+6A`Cqie_;<7<>S6BM&U%b)IwgqEeDd4M>1?N zVcG6vF&^6>HNPxtW<-l9A%nWD@fi2Sab&oUSwgXR@hco&t59P9$zZy=F(RHr~`SskSUfi3&8LbDD)tmGt zAp+L1L%=%eT!VnM#l8Uo7JsGrT8+iD9tM_D0?eQyhd6_9Yw1W5(>d#+_E2nalqO%=IH1uKiy^1mJn%dv42**D;@9F1)` zTyEoG1r0vG2_RO`Z#_gTN+k|U6m#efFtK`*|20so-eh@f-rZjL|*>4NFPN4oso7WYpX-AlL*3m@Q+G(9_njxZJNV(5^VMAB2TP0ykJW zKdjIKVhUiqdI(1Wz;i8*7l&+mO4%cS?_rO#1k6iqZ7-Mq#IvJm}sQ<;%a)!EQO8~%nX#}tv7{3qMj38{d z1|E!qHKv>@g_1YK?YDydVyFivdI!i~znaIg(qMm<4@gVd+Cj%FhSWJfq;?X4HV~x0tTmX zD%A^azV{MfLoh}(_sE|m(ad`GkOzH{6w{-Tp1F#Q&!}GPztS%^M_`trJw!k+dgHp3#(ZZ*mwc{y@Y$;z_YM*Bd_2p6Q&7wyxzEj+I9*?lsZP zv|A9K#4&_*q3tn*rpXWE?h($EL3h~;B9dqoNKChZI8$mokG2q|X_*^z1q%F3@1?cG z>{z-b%vnqFE|l`VzbW=F*F-u~+P3idj-bwzXv+jdhyN5<679?{ksITk>3g+!J7I!a z?^i>z2IQF%PGX+vBd(^#En}#QassQg4(ORDls*}Sxzo!Wl$`vy0{6@ck$in9@|kC> zHKPGXE5OfI5b$R88vL_xK*15J{n&iO>&pDd>`rm=c`a-o&sbnkd2honb$p1zjIDj; zTs`$|pwL1eZ&-{rH`YIJ4`P3gHYqFiFCo9$Kj7bsyeG(LvIWAB>;~rsvDgO&@%H*+ z3pb6k6lP-X0vWA0iE01`0J&dzdb$i4jlA7JL_gGU6{(9Pq+6)g`m-x1kS@2wbyG?b zoZ2G58vwhtd9o1#N*JPucs074(uttUBqB<~yDe`WISUJTb{NzPuj(4rs`<`gB(%J{ zF{-Yc-<7E~m!usJUq1}uEx`cOUF&1KagmTCIZuGM>ou$KVEv?l2b@8xJ=<*)C-@Et z5$EXke8PO&3kJ9f>P;z|A_P|Jsc&jvy=f~Io7u2Xmyk7H0`}%RB1e!Ln+xt@jd1kYei|SetaT zC|vnw$gKnD2+L6v^vyRjj5_p=DoM)cI#k14EsNc1r1*oQKVVC{7;{SV{YbU~y4@ZD{f0v@tIOBpT5WD`=cH6HQ38bI5UQ zCeU$SfAn=`vMqX?fBe3oROSkXz7LgQ(c2J2O3sB(9SE1FrLRH}k08^>jeO>Y!VT|g z;|=&kONRBek9xUDy_2Hl5FhcsmS$XQMk6&j5I9%sWYmPl&~odmh|+3E2q|`4#^FF( z$`Jw0C0blFRea1p5LN1sfbuupq-|E;HUhzXD>>WW}B|0J?MXjG^xM@gBu&!0zPtm2z!6FytD& z^h6oj{>lPCSll~5iD?JuZq#_n*)`yIt0&0EZOPf;E+5D3NP2!HSwG;R^x|j1 zrp^v%xd!@83jh_Aj;c|3a>hM@O|AwTyPN16-}6Mz)dGRX%7)NE*l}AO%I&xpVC?A{ zLxL;-%11Ka4pWshwFIMgwvP4u{D@g=&Y3+^wIXN&?nM?U0lF88bv@ucD_QNCK;P2` z9B&A}mnf8KG}NIa0rlV>6oIeN#eHcJQbc+jXfgX71fSise&G>2w2YA1Q(q)w6WK`| zKB>-hmyditYDor`hcYXW`20(sg_nlp8_KZ9SRacIY~3J@{ehYO)kuAm^2-3b?$$7T zINNc^b9S-Z1ci@`+(0Nltla)u_<_0v>?-6k3sw6=Km^;AZ3!7K&p(f^H#MLHA$QhB zD+$}};Cehk>BT|V$!;ME`X%2>VG7bdHnuB0ccr-E(_Y>J7L1|_z8#wfq8n;+4~L>j z(?n~YI~nHV~3tL1iQk?-F zTnSp3=qxJ=CS`rx!UT~up$i*5Tl26Czc6X!QRw>92cN9QF>JiBM_4|OY?p#EO!VN{ zSnCuk(XI_>m}u0*G)y#E1#8%7-jbjZ3T{-F#4O-Vy(8-~(AcjOib5044RFF02#1ZA zi!^Tma+n}v%=-E`hYcFbX4J!a-Bqa1 zHwZ~TFiuCzWl;-_##wMg@)?xm+y?!SByKR2A6|6rF`Jg)HtO08qw6DrRPb+Ow?@C+ z{7y#tVnhpO0=jKt)@{+kCT0p(sbmMJSM<@S7nxiWDg-c&-ub%V*qqKLBN-c@^5JJe z-huXrNhM;tX_RwgK*uOZvGkg!ElpI%sFRIsv~!r_S$9Z4ck}|WJoquKh42`$WNAd> zKu{Od6F5-6sAD`98!ek8Q)0ssPRyf~gGj}>s2)i`K1K{BN8&d+7WXmg6A!{h%lPR20f7^9+~+J5Mzmct>7Ypi*y~Vi3N~1 zzvRKlODjc}BSt2=A6h)_WuTFXKG)+$COWMN9VtEOfv^FkB93aGyy6q_BZ|*$x>(8P`m^aFe(k{IX5Jt!@S#>qB5h zk;mFFnEHCN=$I)Fa&vKooI@(pI(`;KA)jteMibShwiZu~sg?)PgllQ2sVCEqUGZgt zj38oq(-_>lXpj2 zm80`ByrN=2GPw>ZaYnXsi4ovAEtpI%PN!=}4WLY8yhw_F?f@&(zf`N&T$Qj@KAg@` zsT6q?Cvo`Q*0~XI5a)ZYqz%7&a&8RQfMz;N&H)9Tmg8o&gkbS2-3g}W0?tsrC@ef} zG9aM7pp;|sB}+Ogb3*74G^tv0wH_i!kJ56HM$t?#ZGkjBx00bw&44*Jl~P=tgTx7E zgKaXyoySPWUcF4;8)bj~Nsn8%7zjXwdks1KyF(SVX7@(Wnx52$V{3X^ z9M1Ju$RIAUU#8T^>pSMaLTt8aHZV5h2BC&qd3@bK*=*>Fc4@?INHx1qfq84j4>!Zw zbPF%IDpm^ii(s7H1aQ-z#U|aXh1`@k>S%f&g1QMh|Chm;#pI+BkKwL@5b~xdoJe*V zSj|-+oP6b-VQ`8f-VB11@4qtyPI*6d^}_);$=99XZwg;7CbR{?xGb!xs`cSF$<^JU zZwgB*z&H6O!g0W=e=zo@0)l-w^d^v!*kylb>#9$13XOK$|C> zO&CBlToYx}wO7JNW~~t*!aWKyhd8>jXg+nZGLMlX8qN#2fG= z-Jr(h04kMu)_x)M0lpCnH$6V~$dhA$r7r+Oh+g?jp899Dmwx`S+B^eKncI%cmruiY zGF^N*n$Pi->n!FA9s%jok_7fAIdSj8BcL=Bc^K_KHtc&O%rksz7{GGm>qhjeYPz#4 z*Ezo&0YKAyX)V7c^Zwi227UM9*5{|lTj13-v3P*HC9wJ}NzAU!<+~B_J=kxxg&9B~ zNzZ@&ad!`2Li?E!7)B!_$tS0XzX2SIAa-#$J5=YRdT^M84M?;ijDSU%PN{nKCbUwjcm2)o?J3jj)QVVWVNCb=d?tEKo) gfBm0$!8POrvGw3rfAq)f_51kmAMx+=f8cNb17h3yE&u=k literal 0 HcmV?d00001 diff --git a/docs/source/tutorials/sg_execution_times.rst b/docs/source/tutorials/sg_execution_times.rst new file mode 100644 index 000000000..0bb42ae96 --- /dev/null +++ b/docs/source/tutorials/sg_execution_times.rst @@ -0,0 +1,40 @@ + +:orphan: + +.. _sphx_glr_tutorials_sg_execution_times: + + +Computation times +================= +**00:01.107** total execution time for 2 files **from tutorials**: + +.. container:: + + .. raw:: html + + + + + + + + .. list-table:: + :header-rows: 1 + :class: table table-striped sg-datatable + + * - Example + - Time + - Mem (MB) + * - :ref:`sphx_glr_tutorials_template_tutorial.py` (``template_tutorial.py``) + - 00:01.101 + - 0.0 + * - :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` (``rl_fundamentals_forge_tutorial.py``) + - 00:00.006 + - 0.0 diff --git a/docs/source/tutorials/template_tutorial.codeobj.json b/docs/source/tutorials/template_tutorial.codeobj.json new file mode 100644 index 000000000..21bdbcd5d --- /dev/null +++ b/docs/source/tutorials/template_tutorial.codeobj.json @@ -0,0 +1,27 @@ +{ + "torch.rand": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "builtin_function_or_method" + }, + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "rand" + } + ], + "x": [ + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "Tensor" + } + ] +} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.ipynb b/docs/source/tutorials/template_tutorial.ipynb new file mode 100644 index 000000000..ae808fece --- /dev/null +++ b/docs/source/tutorials/template_tutorial.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n# Template Tutorial\n\n**Author:** [FirstName LastName](https://github.com/username)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Item 1\n * Item 2\n * Item 3\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0\n * GPU ???\n * Other items 3\n\n\nTo test your tutorial locally, you can do one of the following:\n\n* You can control specific files that generate the results by using\n ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable\n respects regular expressions.\n For example to run only ``neural_style_transfer_tutorial.py``,\n use the following command:\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" make html\n```\n or\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" sphinx-build . _build\n```\n* Make a copy of this repository and add only your\n tutorial to the `beginner_source` directory removing all other tutorials.\n Then run ``make html``.\n\nVerify that all outputs were generated correctly in the created HTML.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n\nDescribe Why is this topic important? Add Links to relevant research papers.\n\nThis tutorial walks you through the process of....\n\n## Steps\n\nExample code (the output below is generated automatically):\n\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import torch\n\nx = torch.rand(5, 3)\nprint(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## (Optional) Additional Exercises\n\nAdd additional practice exercises for users to test their knowledge.\nExample: [NLP from Scratch](https://pytorch.org/tutorials/intermediate/char_rnn_generation_tutorial.html#exercises)_.\n\n\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n\nSummarize the steps and concepts covered. Highlight key takeaways.\n\n## Further Reading\n\n* Link1\n* Link2\n\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.py b/docs/source/tutorials/template_tutorial.py new file mode 100644 index 000000000..4018aa1b1 --- /dev/null +++ b/docs/source/tutorials/template_tutorial.py @@ -0,0 +1,91 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Template Tutorial +================= + +**Author:** `FirstName LastName `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Item 1 + * Item 2 + * Item 3 + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0 + * GPU ??? + * Other items 3 + + +To test your tutorial locally, you can do one of the following: + +* You can control specific files that generate the results by using + ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable + respects regular expressions. + For example to run only ``neural_style_transfer_tutorial.py``, + use the following command: + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html + + or + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build + +* Make a copy of this repository and add only your + tutorial to the `beginner_source` directory removing all other tutorials. + Then run ``make html``. + +Verify that all outputs were generated correctly in the created HTML. +""" + +######################################################################### +# Overview +# -------- +# +# Describe Why is this topic important? Add Links to relevant research papers. +# +# This tutorial walks you through the process of.... +# +# Steps +# ----- +# +# Example code (the output below is generated automatically): +# +import torch + +x = torch.rand(5, 3) +print(x) + +###################################################################### +# (Optional) Additional Exercises +# ------------------------------- +# +# Add additional practice exercises for users to test their knowledge. +# Example: `NLP from Scratch `__. +# + +###################################################################### +# Conclusion +# ---------- +# +# Summarize the steps and concepts covered. Highlight key takeaways. +# +# Further Reading +# --------------- +# +# * Link1 +# * Link2 diff --git a/docs/source/tutorials/template_tutorial.py.md5 b/docs/source/tutorials/template_tutorial.py.md5 new file mode 100644 index 000000000..60b156970 --- /dev/null +++ b/docs/source/tutorials/template_tutorial.py.md5 @@ -0,0 +1 @@ +1d0dbb374fded9aaf6cf60085291fe43 \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.rst b/docs/source/tutorials/template_tutorial.rst new file mode 100644 index 000000000..e4768c072 --- /dev/null +++ b/docs/source/tutorials/template_tutorial.rst @@ -0,0 +1,152 @@ + +.. DO NOT EDIT. +.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. +.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: +.. "tutorials/template_tutorial.py" +.. LINE NUMBERS ARE GIVEN BELOW. + +.. only:: html + + .. note:: + :class: sphx-glr-download-link-note + + :ref:`Go to the end ` + to download the full example code. + +.. rst-class:: sphx-glr-example-title + +.. _sphx_glr_tutorials_template_tutorial.py: + + +Template Tutorial +================= + +**Author:** `FirstName LastName `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Item 1 + * Item 2 + * Item 3 + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0 + * GPU ??? + * Other items 3 + + +To test your tutorial locally, you can do one of the following: + +* You can control specific files that generate the results by using + ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable + respects regular expressions. + For example to run only ``neural_style_transfer_tutorial.py``, + use the following command: + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html + + or + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build + +* Make a copy of this repository and add only your + tutorial to the `beginner_source` directory removing all other tutorials. + Then run ``make html``. + +Verify that all outputs were generated correctly in the created HTML. + +.. GENERATED FROM PYTHON SOURCE LINES 56-68 + +Overview +-------- + +Describe Why is this topic important? Add Links to relevant research papers. + +This tutorial walks you through the process of.... + +Steps +----- + +Example code (the output below is generated automatically): + + +.. GENERATED FROM PYTHON SOURCE LINES 68-73 + +.. code-block:: Python + + import torch + + x = torch.rand(5, 3) + print(x) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + tensor([[0.0394, 0.0994, 0.5448], + [0.9781, 0.6839, 0.4422], + [0.9133, 0.0060, 0.0412], + [0.7065, 0.5727, 0.8577], + [0.4415, 0.9282, 0.3205]]) + + + + +.. GENERATED FROM PYTHON SOURCE LINES 74-80 + +(Optional) Additional Exercises +------------------------------- + +Add additional practice exercises for users to test their knowledge. +Example: `NLP from Scratch `__. + + +.. GENERATED FROM PYTHON SOURCE LINES 82-92 + +Conclusion +---------- + +Summarize the steps and concepts covered. Highlight key takeaways. + +Further Reading +--------------- + +* Link1 +* Link2 + + +.. rst-class:: sphx-glr-timing + + **Total running time of the script:** (0 minutes 1.101 seconds) + + +.. _sphx_glr_download_tutorials_template_tutorial.py: + +.. only:: html + + .. container:: sphx-glr-footer sphx-glr-footer-example + + .. container:: sphx-glr-download sphx-glr-download-jupyter + + :download:`Download Jupyter notebook: template_tutorial.ipynb ` + + .. container:: sphx-glr-download sphx-glr-download-python + + :download:`Download Python source code: template_tutorial.py ` + + .. container:: sphx-glr-download sphx-glr-download-zip + + :download:`Download zipped: template_tutorial.zip ` diff --git a/docs/source/tutorials/template_tutorial.zip b/docs/source/tutorials/template_tutorial.zip new file mode 100644 index 0000000000000000000000000000000000000000..321b881c49affa7b268465506f2f059b02457417 GIT binary patch literal 5757 zcmd5=-EJGl6;_kBKo`AC(c6O+3P^%TSyBtMOdLmb9?n!cPc;CvYn)n#Vjeq)8KRR^zreF(_?~A@Ca0L)_Tg7lNr>Xk{w~~LYj9{cB3ec z_igtF6YCo~Ax3fd(6Xh!ZLKKotMDy^;#QU=9?d^?~pVY%{W{bTV zrW&mJG)Rso zo~E{@O9g*wDVzz6z^j*OQB^Z8#JoDClTQArRu$M+Gl5vQLr)=%_S=2U?}L@!hwq}H zR+aD6#yUhvwmSB&iDfe}-d(%?^vt?!LKg?!2i*q?t-k30o*q4Vv>?1h+#Rv&DuTAr znI+g6+=G;9;z-(NLhI>n@LDDe<(90Gvs{o!BCYMEGUFcNOY{|7%ZuQD@2sY(l$jc- zEM!%M?1B5o(nyE24rz&8uC=ddIHkG*8&)q(zc@NMIezt3(my&oJAU;dO-Y&y<*X^B z@pK_jN<%HX$bCsGYv;4f3EZIo*gH9{wQwY_5Y(zt)}W}c;E83LK$SurORhFBPEVnN zk+l;#2_=(wCrmYxZn=)rbeB0GL4zT4N5J-?K>flYm_~+jQFey#<2kaknzV_uqW8o2 zPKhxU;#|_i7dkMp?tdVnDksWZb%wRlIdv%s4;odqe10c|LO= zD$K$9c@n|3+=J4g94mw9C$oN)F7*~C3wwbCAY>cakecoG5GsV)g!rWC+zV-n_VWkn z)Mz?Ot+2QDWsQ(uqNL`zmP2pHZ49w(dRyisk}#()&z_%jLlcWO*EP^iUtXX`D|v~> zPBTKg{#;g>Q$vYX1XZ{s_`{Z{5ml7jv5bF2NATo{GUq&nG)V^ z`+w^yx)d6lxWoC0v-Nlq9A7${A?;DLU5v2p)XTDJl?#;PrVhd(vc>VT!IduNh%lcS;i*2~hP zn1$YEm#Nt^9bZY8sR~Vd;ruoTE8oF#_O@7TFP-2cM4~e^e9-0`SGo)F2wf8XQI5`y zz0@)v;~d zmsun|a0z`0bfW{pIhQC+bPjPTrcJ^=sa+`VS5oBM@o&T^2-^wyzmM73(?QgqL#=-U zYW?#cr~b2_Zfx-X&u@iVs+^kPU(kl)Oln=lJ$lU@nV7^QkP|;GC8lD|`Z#oSo`Q$7 znzJd%fRa2Gm|E-kbOFqk3ok%#xiLSSg`x!P0(m5KB^yCs&-cv=DyZ^iPMq~Z?3t+- zRQm;rtJKzOHAKJC%E-83_9lEcEL-8%O+k>(b9-qP<}B;G(B?M50|P|43g!l8V3xt# z>zm;1&DLAM+cq;;0&o12&0Gh3n{fiOI}lPU6DwN;H{aSqxK+YoyXozU-i_ksjNeDe z>p*VKVuudQ?E$D4vzNhZE zxxjU-{5r85^9!%?bI-pEl;d4cDu6kDz6XZm?eFEt|F08s8Go`Nw;|AhX(8zMmoIl9 zR91Bz@N*t|z=&1!xj38Gi$1rL?t3TAeXt1DW)JtIFw^FM_H> z^P3Q_Xbf2Ly~`1TwA#9i8r``%>^x4pRVJ|5qGbB Date: Tue, 7 Oct 2025 11:46:42 -0700 Subject: [PATCH 03/33] Add API docs for actor/service --- docs/source/api.md | 48 +++++++------ docs/source/api_actors.md | 58 ++++------------ docs/source/api_core.md | 68 ------------------- docs/source/api_data.md | 63 ----------------- docs/source/api_data_models.md | 51 -------------- docs/source/api_losses.md | 27 -------- docs/source/api_service.md | 21 ++++++ docs/source/api_types.md | 41 ------------ docs/source/api_util.md | 89 ------------------------- docs/source/conf.py | 80 +++++++--------------- src/forge/controller/actor.py | 77 +++++++++++++++------ src/forge/controller/service/service.py | 11 +++ 12 files changed, 148 insertions(+), 486 deletions(-) delete mode 100644 docs/source/api_core.md delete mode 100644 docs/source/api_data.md delete mode 100644 docs/source/api_data_models.md delete mode 100644 docs/source/api_losses.md create mode 100644 docs/source/api_service.md delete mode 100644 docs/source/api_types.md delete mode 100644 docs/source/api_util.md diff --git a/docs/source/api.md b/docs/source/api.md index e4c2d064e..f4ecd01dc 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -1,34 +1,32 @@ # API Reference -This API reference is organized by priority of concepts that users should be exposed to, based on how they're used in the core Forge workflows. - -```{eval-rst} -.. toctree:: - :maxdepth: 2 - - api_core - api_actors - api_data_models - api_types - api_util - api_data - api_losses -``` +This section provides comprehensive API documentation for TorchForge. ## Overview -The Forge API is structured around key concepts in order of priority: +TorchForge is a PyTorch native platform for post-training generative AI models, +designed to streamline reinforcement learning workflows for large language +models. The platform leverages PyTorch's distributed computing capabilities +and is built on top of [Monarch](https://meta-pytorch.org/monarch/), +making extensive use of actors for distributed computation and fault tolerance. + +Key Features of TorchForge include: -1. **[Core Concepts](api_core.md)** - Actor System, ForgeActor, and ForgeService fundamentals -2. **[Built-in Actors](api_actors.md)** - Policy, Trainer, ReplayBuffer, and ReferenceModel -3. **[Data Models](api_data_models.md)** - Completion, Prompt, Episode, and other data structures -4. **[Configuration and Types](api_types.md)** - Core types and configuration classes -5. **[Utilities](api_util.md)** - Distributed computing, logging, and observability tools -6. **[Data Processing](api_data.md)** - Rewards, tokenization, and data handling utilities -7. **[Loss Functions](api_losses.md)** - GRPO and REINFORCE loss implementations +- **Actor-Based Architecture**: TorchForge uses an actor-based system for distributed training, providing excellent scalability and fault tolerance. +- **PyTorch Native**: Built natively on PyTorch, ensuring seamless integration with existing PyTorch workflows. +- **Post-Training Focus**: Specifically designed for post-training techniques like RLHF, SFT, and other alignment methods. +- **Distributed by Design**: Supports multi-GPU and multi-node training out of the box. -## Quick Start -To get started with Forge, begin with the [Core Concepts](api_core.md) to understand the actor system foundation, then explore the [Built-in Actors](api_actors.md) for common RL workflows. +For most use cases, you'll interact with the high-level service +interfaces, which handle the complexity of actor coordination and +distributed training automatically. -For a practical example, see the GRPO implementation in `apps/grpo/main.py` which demonstrates how these components work together in a complete reinforcement learning training loop. +For advanced users who need fine-grained control, the individual actor +APIs provide direct access to the underlying distributed components. + +```{toctree} +:maxdepth: 1 +api_actors +api_service +``` diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index 1be09712d..f086591d8 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -1,54 +1,20 @@ -# Built-in Actors - -On top of the services/actors foundation, Forge provides implementations of actors that are useful in RL workflows. - -## Policy - -Inference and generation via vLLM. The {class}`forge.actors.policy.Policy` is a key actor for generating completions from language models. +# ForgeActor ```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.policy.Policy - forge.actors.policy.PolicyWorker - forge.actors.policy.EngineConfig - forge.actors.policy.SamplingConfig +.. currentmodule:: forge.actors ``` -## Trainer - -Training via torchtitan. The {class}`forge.actors.trainer.RLTrainer` handles reinforcement learning training loops. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.trainer.RLTrainer -``` - -## ReplayBuffer - -For storing experience and sampling to the trainer - the glue between policy and trainer. The {class}`forge.actors.replay_buffer.ReplayBuffer` manages experience data for RL training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.replay_buffer.ReplayBuffer -``` - -## ReferenceModel - -Used for RL correctness. The {class}`forge.actors.reference_model.ReferenceModel` provides reference logprobs for RL algorithms. +The actors module contains the core components for model training +and inference in TorchForge. These pre-built actors provide essential +functionality for reinforcement learning workflows and can be used +as building blocks for complex distributed training systems. ```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: +.. currentmodule:: forge.controller.actor - forge.actors.reference_model.ReferenceModel +.. autoclass:: ForgeActor + :members: + :undoc-members: + :show-inheritance: + :exclude-members: logger, setup, set_env ``` diff --git a/docs/source/api_core.md b/docs/source/api_core.md deleted file mode 100644 index 785302ad3..000000000 --- a/docs/source/api_core.md +++ /dev/null @@ -1,68 +0,0 @@ -# Core Concepts - -## Actor System - -Forge is built on top of Monarch and makes extensive use of actors. -Actors are the foundation for building distributed, fault-tolerant systems. - -## ForgeActor - -In Forge, everything centers around the {class}`forge.controller.actor.ForgeActor`, which is a customized version of a Monarch actor tailored for Forge-specific needs. - -The {class}`forge.controller.actor.ForgeActor` differs from a standard Monarch actor by allowing you to specify resource requirements using the `options()` method. This API lets you define the resources your actor needs and create two types of constructs: - -- A regular Monarch actor using `as_actor()` -- A service using `as_service()` - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.controller.actor.ForgeActor -``` - -### Resource Configuration Example - -Options are important because they demonstrate the resource requirements and how we represent them in Forge: - -```python -from forge.controller.actor import ForgeActor - -class MyActor(ForgeActor): - pass - -# Create a service with specific resource requirements -service = MyActor.options( - hosts=1, - procs=8, - replicas=1, - with_gpus=True -).as_service() -``` - -This example creates a service that has 1 replica, where the replica consists of 1 remote host and 8 processes, using Monarch's remote allocations. - -### Key Methods - -**`options()`** -Configures resource requirements for the actor. - -**`setup()`** -Sets up an actor. All actors should implement this for any heavyweight setup (like PyTorch distributed initialization, model checkpoint loading, etc.). - -**`launch()`** -Logic to provision and deploy a new replica. This is what services use to spin up replicas. - -## ForgeService - -Services are replicated, fault-tolerant versions of actors. They provide high availability and load distribution across multiple actor instances. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.controller.service.Service - forge.controller.service.ServiceInterface -``` diff --git a/docs/source/api_data.md b/docs/source/api_data.md deleted file mode 100644 index a652c6d0a..000000000 --- a/docs/source/api_data.md +++ /dev/null @@ -1,63 +0,0 @@ -# Data Processing - -Data handling utilities for datasets, rewards, and tokenization. - -## Rewards - -Reward functions for RL training. The {mod}`forge.data.rewards` module provides reward functions like {class}`forge.data.rewards.MathReward` and {class}`forge.data.rewards.ThinkingReward`. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.rewards -``` - -## Tokenization - -Tokenization utilities for processing text data. The {mod}`forge.data.tokenizer` module provides tokenization functions and utilities. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.tokenizer -``` - -## Data Collation - -Data collation utilities for batching and processing. The {mod}`forge.data.collate` module provides functions for collating data into batches. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.collate -``` - -## Data Sharding - -Data sharding utilities for distributed processing. The {mod}`forge.data.sharding` module provides sharding strategies for distributed training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.sharding -``` - -## Data Utilities - -General data processing utilities. The {mod}`forge.data.utils` module provides miscellaneous data handling functions. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.utils -``` diff --git a/docs/source/api_data_models.md b/docs/source/api_data_models.md deleted file mode 100644 index a7c02caa0..000000000 --- a/docs/source/api_data_models.md +++ /dev/null @@ -1,51 +0,0 @@ -# Data Models - -Base data models for common RL workflows. - -## Completion - -Outputs from vLLM. The {class}`forge.data_models.completion.Completion` represents a model-generated completion for a given prompt. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.completion.Completion -``` - -## Prompt - -Input prompts for models. The {class}`forge.data_models.prompt.Prompt` encapsulates prompt data for language models. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.prompt.Prompt -``` - -## Episode - -Training episodes for RL. The {class}`forge.data_models.episode.Episode` represents a complete interaction episode in reinforcement learning. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.episode.Episode -``` - -## ScoredCompletion - -Completions with associated scores. The {class}`forge.data_models.scored_completion.ScoredCompletion` extends completions with scoring information. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.scored_completion.ScoredCompletion -``` diff --git a/docs/source/api_losses.md b/docs/source/api_losses.md deleted file mode 100644 index d16f2a6fc..000000000 --- a/docs/source/api_losses.md +++ /dev/null @@ -1,27 +0,0 @@ -# Loss Functions - -Loss functions for RL training. - -## GRPO Loss - -Group Relative Policy Optimization loss function. The {mod}`forge.losses.grpo_loss` module provides loss functions for GRPO training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.losses.grpo_loss -``` - -## REINFORCE Loss - -REINFORCE algorithm loss function. The {mod}`forge.losses.reinforce_loss` module provides loss functions for REINFORCE training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.losses.reinforce_loss -``` diff --git a/docs/source/api_service.md b/docs/source/api_service.md new file mode 100644 index 000000000..878d43327 --- /dev/null +++ b/docs/source/api_service.md @@ -0,0 +1,21 @@ +# Service + +```{eval-rst} +.. currentmodule:: forge.controller.service.service +``` + +```{eval-rst} +.. autoclass:: Service + :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop + :show-inheritance: + :special-members: __init__ +``` + +## ServiceActor + +```{eval-rst} +.. autoclass:: ServiceActor + :members: call, call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop + :show-inheritance: + :special-members: __init__ +``` diff --git a/docs/source/api_types.md b/docs/source/api_types.md deleted file mode 100644 index e7bd423a3..000000000 --- a/docs/source/api_types.md +++ /dev/null @@ -1,41 +0,0 @@ -# Configuration and Types - -## Core Type Definitions - -Core type definitions used throughout Forge. The {mod}`forge.types` module contains fundamental types and configurations. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.types -``` - -## Configuration Classes - -Configuration classes for actors and services. - -### EngineConfig - -Configuration for vLLM engines used in Policy actors. The {class}`forge.actors.policy.EngineConfig` extends vLLM's EngineArgs with worker-specific fields. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.policy.EngineConfig -``` - -### SamplingConfig - -Configuration for sampling parameters in Policy actors. The {class}`forge.actors.policy.SamplingConfig` provides overrides for vLLM's sampling parameters. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.policy.SamplingConfig -``` diff --git a/docs/source/api_util.md b/docs/source/api_util.md deleted file mode 100644 index 06438b358..000000000 --- a/docs/source/api_util.md +++ /dev/null @@ -1,89 +0,0 @@ -# Utilities - -Utility functions and classes for distributed computing, logging, and operations. - -## Distributed Computing - -Utilities for distributed computing operations. The {mod}`forge.util.distributed` module provides functions for setting up distributed environments. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.distributed -``` - -## Logging - -Logging utilities for Forge applications. The {mod}`forge.util.logging` module provides logging configuration and utilities. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.logging -``` - -## Operations - -Core operations and utilities. The {mod}`forge.util.ops` module contains commonly used operations for RL workflows. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.ops -``` - -## Metric Logging - -Metric logging utilities. The {mod}`forge.util.metric_logging` module provides utilities for logging metrics. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.metric_logging -``` - -## Observability - -### Metrics - -Metrics collection and reporting. The {mod}`forge.observability.metrics` module provides the core metrics infrastructure. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.observability.metrics -``` - -### Performance Tracking - -Performance tracking utilities. The {mod}`forge.observability.perf_tracker` module provides performance measurement tools. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.observability.perf_tracker -``` - -### Metric Actors - -Actors for metric collection. The {mod}`forge.observability.metric_actors` module provides actors for distributed metric collection. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.observability.metric_actors -``` diff --git a/docs/source/conf.py b/docs/source/conf.py index 2d1c95e6a..aa73efa62 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,33 +17,10 @@ import pytorch_sphinx_theme2 # Add the source directory to Python path so modules can be imported -sys.path.insert(0, os.path.abspath("../../src/forge")) - - -# Determine the version path for deployment -def get_version_path(): - """Get the version path based on environment variables or git context.""" - # Check if we're in CI/CD and get the target folder - github_ref = os.environ.get("GITHUB_REF", "") - - # Convert refs/tags/v1.12.0rc3 into 1.12. - # Matches the logic in .github/workflows/docs.yml - if github_ref.startswith("refs/tags/v"): - import re - - match = re.match(r"^refs/tags/v([0-9]+\.[0-9]+)\..*", github_ref) - if match: - return match.group(1) + "/" - - # Default to main for main branch or local development - return "main/" - - -# Set base URL based on deployment context -version_path = get_version_path() +sys.path.insert(0, os.path.abspath("../../src")) project = "torchforge" -copyright = "" +copyright = "2025, PyTorch Contributors" author = "PyTorch Contributors" release = "0.1" @@ -61,12 +38,9 @@ def get_version_path(): "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", - "sphinx_gallery.gen_gallery", ] -html_baseurl = ( - f"https://meta-pytorch.org/forge/{version_path}" # needed for sphinx-sitemap -) +html_baseurl = "https://meta-pytorch.org/forge/" # needed for sphinx-sitemap sitemap_locales = [None] sitemap_excludes = [ "search.html", @@ -78,7 +52,7 @@ def get_version_path(): "_templates", os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), ] -exclude_patterns = ["tutorials/index.rst"] +exclude_patterns = [] sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../src")) @@ -116,7 +90,7 @@ def get_version_path(): }, { "name": "PyPi", - "url": "https://pypi.org/project/torchforge/", + "url": "https://pypi.org/project/forge/", "icon": "fa-brands fa-python", }, ], @@ -135,21 +109,9 @@ def get_version_path(): "github_user": "meta-pytorch", "github_repo": "forge", "feedback_url": "https://github.com/meta-pytorch/forge", - "colab_branch": "gh-pages", "github_version": "main", "doc_path": "docs/source", - "has_sphinx_gallery": True, # Enable tutorial call-to-action links -} - -# For tutorial repository configuration -# Note: github_user and github_repo are combined in the template as "{{ github_user }}/{{ github_repo }}" -# So we keep github_user = "meta-pytorch" and github_repo = "forge" already set above -# and only need to ensure the branch settings are correct -tutorial_repo_config = { - "github_version": "main", # This maps to github_branch in the template - "colab_branch": "gh-pages", } -html_context.update(tutorial_repo_config) myst_enable_extensions = [ "colon_fence", @@ -157,17 +119,25 @@ def get_version_path(): "html_image", ] -# -- Sphinx Gallery configuration ------------------------------------------- -sphinx_gallery_conf = { - "examples_dirs": "tutorial_sources", # Path to examples directory - "gallery_dirs": "tutorials", # Path to generate gallery - "filename_pattern": ".*", # Include all files - "download_all_examples": False, - "first_notebook_cell": "%matplotlib inline", - "plot_gallery": "True", - "promote_jupyter_magic": True, - "backreferences_dir": None, - "write_computation_times": True, - "show_signature": False, +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "special-members": "__init__", + "undoc-members": False, # Changed to False to prevent private members + "exclude-members": "__weakref__", } +# Skip private members (starting with _) by default +def autodoc_skip_member(app, what, name, obj, skip, options): + """Skip private members (starting with _) but keep special methods (__*).""" + # Skip private attributes and methods but keep special methods like __init__ + if name.startswith('_') and not name.startswith('__'): + return True + return skip + +def setup(app): + app.connect('autodoc-skip-member', autodoc_skip_member) + +# Autosummary settings +autosummary_generate = True +autosummary_imported_members = True diff --git a/src/forge/controller/actor.py b/src/forge/controller/actor.py index a899da6f0..1bc63a0e0 100644 --- a/src/forge/controller/actor.py +++ b/src/forge/controller/actor.py @@ -12,7 +12,7 @@ from monarch.actor import Actor, current_rank, current_size, endpoint -from forge.controller.provisioner import get_proc_mesh, stop_proc_mesh +from forge.controller.proc_mesh import get_proc_mesh, stop_proc_mesh from forge.types import ProcessConfig, ServiceConfig @@ -26,7 +26,6 @@ class ForgeActor(Actor): hosts: int | None = None with_gpus: bool = False num_replicas: int = 1 - mesh_name: str | None = None _extra_config: dict[str, Any] = {} def __init__(self, *args, **kwargs): @@ -59,7 +58,6 @@ def options( hosts: int | None = None, with_gpus: bool = False, num_replicas: int = 1, - mesh_name: str | None = None, **kwargs, ) -> Type[T]: """ @@ -69,34 +67,47 @@ def options( `.as_actor()` or `.as_service()`. Each call creates a separate subclass, so multiple different configurations can coexist without interfering with each other. - ---- Usage Examples ---- + Examples: - # Pre-configure a service with multiple replicas - service = await MyForgeActor.options(num_replicas=2, procs=2).as_service(...) - await service.shutdown() + * Pre-configure a service with multiple replicas: + + .. code-block:: python + + service = await MyForgeActor.options(num_replicas=2, procs=2).as_service(...) + await service.shutdown() - # Default usage without calling options - service = await MyForgeActor.as_service(...) - await service.shutdown() + * Default usage without calling options: - # Pre-configure a single actor - actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) - await actor.shutdown() + .. code-block:: python + + service = await MyForgeActor.as_service(...) + await service.shutdown() - # Default usage without calling options - actor = await MyForgeActor.as_actor(...) - await actor.shutdown() + * Pre-configure a single actor + + .. code-block:: python + + actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) + await actor.shutdown() + + * Default usage without calling options + + .. code-block:: python + + actor = await MyForgeActor.as_actor(...) + await actor.shutdown() """ attrs = { "procs": procs, + "hosts": hosts, "with_gpus": with_gpus, "num_replicas": num_replicas, - "mesh_name": mesh_name, "_extra_config": kwargs, } + return type(cls.__name__, (cls,), attrs) @classmethod @@ -119,12 +130,11 @@ async def as_service( "hosts": cls.hosts, "with_gpus": cls.with_gpus, "num_replicas": cls.num_replicas, - "mesh_name": cls.mesh_name, **cls._extra_config, # all extra fields } cfg = ServiceConfig(**cfg_kwargs) - logger.info("Spawning Service for %s", cls.__name__) + logger.info("Spawning Service Actor for %s", cls.__name__) service = Service(cfg, cls, actor_args, actor_kwargs) await service.__initialize__() return ServiceInterface(service, cls) @@ -144,6 +154,28 @@ async def setup(self): """ pass + @endpoint + async def set_env(self, addr: str, port: str): + """A temporary workaround to set master addr/port. + + TODO - issues/144. This should be done in proc_mesh creation. + The ideal path: + - Create a host mesh + - Grab a host from host mesh, from proc 0 spawn an actor that + gets addr/port + - Spawn procs on the HostMesh with addr/port, setting the + addr/port in bootstrap. + + We can't currently do this because HostMesh only supports single + proc_mesh creation at the moment. This will be possible once + we have "proper HostMesh support". + + """ + import os + + os.environ["MASTER_ADDR"] = addr + os.environ["MASTER_PORT"] = port + @classmethod async def launch(cls, *args, **kwargs) -> "ForgeActor": """Provisions and deploys a new actor. @@ -163,14 +195,17 @@ async def launch(cls, *args, **kwargs) -> "ForgeActor": procs=cls.procs, hosts=cls.hosts, with_gpus=cls.with_gpus, - mesh_name=cls.mesh_name, ) proc_mesh = await get_proc_mesh(process_config=cfg) actor_name = kwargs.pop("name", cls.__name__) - actor = proc_mesh.spawn(actor_name, cls, *args, **kwargs) + actor = await proc_mesh.spawn(actor_name, cls, *args, **kwargs) actor._proc_mesh = proc_mesh + + if hasattr(proc_mesh, "_hostname") and hasattr(proc_mesh, "_port"): + host, port = proc_mesh._hostname, proc_mesh._port + await actor.set_env.call(addr=host, port=port) await actor.setup.call() return actor diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index 0b655fb6a..402b9a419 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -1159,3 +1159,14 @@ def _get_internal_state(self) -> dict: def __repr__(self): return f"Service(actor={self._actor_def.__name__})" + + +# Copy docstrings from Service to ServiceActor methods for Sphinx autodoc +# This ensures ServiceActor methods have complete docstrings while avoiding duplication +ServiceActor.call.__doc__ = Service._call.__doc__ +ServiceActor.call_all.__doc__ = Service.call_all.__doc__ +ServiceActor.start_session.__doc__ = Service.start_session.__doc__ +ServiceActor.get_metrics.__doc__ = Service.get_metrics.__doc__ +ServiceActor.get_metrics_summary.__doc__ = Service.get_metrics_summary.__doc__ +ServiceActor.terminate_session.__doc__ = Service.terminate_session.__doc__ +ServiceActor.stop.__doc__ = Service.stop.__doc__ From 23dda071f38886d8cd68048a4905d457e1cabd99 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 11:48:55 -0700 Subject: [PATCH 04/33] Update --- .gitignore | 2 +- docs/source/conf.py | 77 +- ...r_rl_fundamentals_forge_tutorial_thumb.png | Bin 28056 -> 0 bytes .../sphx_glr_template_tutorial_thumb.png | Bin 28056 -> 0 bytes docs/source/tutorials/index.rst | 63 -- ...l_fundamentals_forge_tutorial.codeobj.json | 215 ----- .../rl_fundamentals_forge_tutorial.ipynb | 158 ---- .../rl_fundamentals_forge_tutorial.py | 558 ------------- .../rl_fundamentals_forge_tutorial.py.md5 | 1 - .../rl_fundamentals_forge_tutorial.rst | 788 ------------------ .../rl_fundamentals_forge_tutorial.zip | Bin 40960 -> 0 bytes docs/source/tutorials/sg_execution_times.rst | 40 - .../tutorials/template_tutorial.codeobj.json | 27 - docs/source/tutorials/template_tutorial.ipynb | 75 -- docs/source/tutorials/template_tutorial.py | 91 -- .../source/tutorials/template_tutorial.py.md5 | 1 - docs/source/tutorials/template_tutorial.rst | 152 ---- docs/source/tutorials/template_tutorial.zip | Bin 5757 -> 0 bytes src/forge/controller/actor.py | 69 +- src/forge/controller/service/service.py | 4 + 20 files changed, 100 insertions(+), 2221 deletions(-) delete mode 100644 docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png delete mode 100644 docs/source/tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png delete mode 100644 docs/source/tutorials/index.rst delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.rst delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.zip delete mode 100644 docs/source/tutorials/sg_execution_times.rst delete mode 100644 docs/source/tutorials/template_tutorial.codeobj.json delete mode 100644 docs/source/tutorials/template_tutorial.ipynb delete mode 100644 docs/source/tutorials/template_tutorial.py delete mode 100644 docs/source/tutorials/template_tutorial.py.md5 delete mode 100644 docs/source/tutorials/template_tutorial.rst delete mode 100644 docs/source/tutorials/template_tutorial.zip diff --git a/.gitignore b/.gitignore index 413066489..c952405d6 100644 --- a/.gitignore +++ b/.gitignore @@ -153,7 +153,7 @@ docs/source/generated_examples/ docs/source/gen_modules/ docs/source/generated/ docs/source/sg_execution_times.rst -docs/source/tutorials +docs/source/tutorials/* # pytorch-sphinx-theme gets installed here docs/src diff --git a/docs/source/conf.py b/docs/source/conf.py index aa73efa62..77ee13383 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,10 +17,33 @@ import pytorch_sphinx_theme2 # Add the source directory to Python path so modules can be imported -sys.path.insert(0, os.path.abspath("../../src")) +sys.path.insert(0, os.path.abspath("../../src/forge")) + + +# Determine the version path for deployment +def get_version_path(): + """Get the version path based on environment variables or git context.""" + # Check if we're in CI/CD and get the target folder + github_ref = os.environ.get("GITHUB_REF", "") + + # Convert refs/tags/v1.12.0rc3 into 1.12. + # Matches the logic in .github/workflows/docs.yml + if github_ref.startswith("refs/tags/v"): + import re + + match = re.match(r"^refs/tags/v([0-9]+\.[0-9]+)\..*", github_ref) + if match: + return match.group(1) + "/" + + # Default to main for main branch or local development + return "main/" + + +# Set base URL based on deployment context +version_path = get_version_path() project = "torchforge" -copyright = "2025, PyTorch Contributors" +copyright = "" author = "PyTorch Contributors" release = "0.1" @@ -38,9 +61,12 @@ "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinx_gallery.gen_gallery", ] -html_baseurl = "https://meta-pytorch.org/forge/" # needed for sphinx-sitemap +html_baseurl = ( + f"https://meta-pytorch.org/forge/{version_path}" # needed for sphinx-sitemap +) sitemap_locales = [None] sitemap_excludes = [ "search.html", @@ -52,7 +78,7 @@ "_templates", os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), ] -exclude_patterns = [] +exclude_patterns = ["tutorials/index.rst", "tutorials/template_tutorial.rst"] sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../src")) @@ -90,7 +116,7 @@ }, { "name": "PyPi", - "url": "https://pypi.org/project/forge/", + "url": "https://pypi.org/project/torchforge/", "icon": "fa-brands fa-python", }, ], @@ -109,9 +135,21 @@ "github_user": "meta-pytorch", "github_repo": "forge", "feedback_url": "https://github.com/meta-pytorch/forge", + "colab_branch": "gh-pages", "github_version": "main", "doc_path": "docs/source", + "has_sphinx_gallery": True, # Enable tutorial call-to-action links +} + +# For tutorial repository configuration +# Note: github_user and github_repo are combined in the template as "{{ github_user }}/{{ github_repo }}" +# So we keep github_user = "meta-pytorch" and github_repo = "forge" already set above +# and only need to ensure the branch settings are correct +tutorial_repo_config = { + "github_version": "main", # This maps to github_branch in the template + "colab_branch": "gh-pages", } +html_context.update(tutorial_repo_config) myst_enable_extensions = [ "colon_fence", @@ -123,21 +161,22 @@ "members": True, "member-order": "bysource", "special-members": "__init__", - "undoc-members": False, # Changed to False to prevent private members + "undoc-members": True, "exclude-members": "__weakref__", } -# Skip private members (starting with _) by default -def autodoc_skip_member(app, what, name, obj, skip, options): - """Skip private members (starting with _) but keep special methods (__*).""" - # Skip private attributes and methods but keep special methods like __init__ - if name.startswith('_') and not name.startswith('__'): - return True - return skip - -def setup(app): - app.connect('autodoc-skip-member', autodoc_skip_member) -# Autosummary settings -autosummary_generate = True -autosummary_imported_members = True +# -- Sphinx Gallery configuration ------------------------------------------- +sphinx_gallery_conf = { + "examples_dirs": "tutorial_sources", # Path to examples directory + "gallery_dirs": "tutorials", # Path to generate gallery + "filename_pattern": ".*", # Include all files + "download_all_examples": False, + "first_notebook_cell": "%matplotlib inline", + "plot_gallery": "True", + "promote_jupyter_magic": True, + "backreferences_dir": None, + "write_computation_times": True, + "show_signature": False, + "write_computation_times": False +} diff --git a/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png b/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png deleted file mode 100644 index 3b362cf91299659eeec19a203a6d16971bc138fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28056 zcmd3NV|OM^w03Me6XT9;+qP}nwr$(CHNnKTZF@4&o9C?a>HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C&HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C& - -.. thumbnail-parent-div-open - -.. raw:: html - -
- -.. only:: html - - .. image:: /tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png - :alt: - - :ref:`sphx_glr_tutorials_template_tutorial.py` - -.. raw:: html - -
Template Tutorial
-
- - -.. raw:: html - -
- -.. only:: html - - .. image:: /tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png - :alt: - - :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` - -.. raw:: html - -
RL Fundamentals Using Forge Terminology
-
- - -.. thumbnail-parent-div-close - -.. raw:: html - - - - -.. toctree:: - :hidden: - - /tutorials/template_tutorial - /tutorials/rl_fundamentals_forge_tutorial - diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json b/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json deleted file mode 100644 index c1c95a0b1..000000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "Any": [ - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "_SpecialForm" - }, - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "Any" - } - ], - "Dict": [ - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "_SpecialGenericAlias" - }, - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "Dict" - } - ], - "MockResponse": [ - { - "is_class": true, - "is_explicit": false, - "module": "__main__", - "module_short": "__main__", - "name": "MockResponse" - } - ], - "Optional": [ - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "_SpecialForm" - }, - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "Optional" - } - ], - "asyncio.gather": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - }, - { - "is_class": false, - "is_explicit": false, - "module": "asyncio", - "module_short": "asyncio", - "name": "gather" - } - ], - "conceptual_rl_step": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_advantages_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_dataset_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_episode_from_response": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_policy_service": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_reference_model_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_replay_buffer_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_reward_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_trainer_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "demonstrate_production_scaling": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "example_experience": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "dict" - } - ], - "show_infrastructure_challenges": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "torch.cat": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "builtin_function_or_method" - }, - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "cat" - } - ], - "torch.tensor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "builtin_function_or_method" - }, - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "tensor" - } - ] -} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb b/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb deleted file mode 100644 index 3981a531b..000000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb +++ /dev/null @@ -1,158 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n# RL Fundamentals Using Forge Terminology\n\n**Author:** [Your Name](https://github.com/yourusername)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Core RL components in Forge (Dataset, Policy, Reward Model, etc.)\n * How RL concepts map to distributed Forge services\n * Building scalable RL training loops with fault tolerance\n * Resource management and independent scaling patterns\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0+\n * GPU access recommended\n * Basic understanding of reinforcement learning\n * Familiarity with async/await in Python\n\nThis tutorial teaches RL fundamentals using Forge's exact terminology and architecture.\nWe'll start with a simple math tutoring example to understand how traditional RL concepts\nmap to Forge's distributed service model.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Core RL Components in Forge\n\nLet's start with a simple math tutoring example to understand RL concepts\nwith the exact names Forge uses. Think of it as teaching an AI student:\n\n- **Dataset**: Provides questions (like \"What is 2+2?\")\n- **Policy**: The AI student being trained (generates answers like \"The answer is 4\")\n- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95)\n- **Reference Model**: Copy of original student (prevents drift from baseline)\n- **Replay Buffer**: Notebook that stores experiences (question + answer + score)\n- **Trainer**: The tutor that improves the student based on experiences\n\nHere's how these components interact in a conceptual RL step:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import asyncio\nfrom typing import Any, Dict, Optional\n\nimport torch\n\n\ndef conceptual_rl_step():\n \"\"\"\n Conceptual example showing the RL learning flow.\n See apps/grpo/main.py for actual GRPO implementation.\n \"\"\"\n # 1. Get a math problem\n question = \"What is 2+2?\" # dataset.sample()\n\n # 2. Student generates answer\n answer = \"The answer is 4\" # policy.generate(question)\n\n # 3. Teacher grades it\n score = 0.95 # reward_model.evaluate(question, answer)\n\n # 4. Compare to original student\n baseline = 0.85 # reference_model.compute_logprobs(question, answer)\n\n # 5. Store the experience\n experience = {\n \"question\": question,\n \"answer\": answer,\n \"score\": score,\n \"baseline\": baseline,\n }\n # replay_buffer.add(experience)\n\n # 6. When enough experiences collected, improve student\n # trainer.train_step(batch) # Student gets better!\n\n return experience\n\n\nexample_experience = conceptual_rl_step()\nprint(\"Example RL experience:\", example_experience)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## From Concepts to Forge Services\n\nHere's the key insight: **Each RL component becomes a Forge service**.\nThe toy example above maps directly to Forge's distributed architecture:\n\n* Dataset \u2192 DatasetActor\n* Policy \u2192 Policy\n* Reward Model \u2192 RewardActor\n* Reference Model \u2192 ReferenceModel\n* Replay Buffer \u2192 ReplayBuffer\n* Trainer \u2192 RLTrainer\n\nLet's see how the conceptual example translates to actual Forge service calls:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]:\n \"\"\"\n RL step using actual Forge service APIs.\n This shows the same logic as conceptual_rl_step but with real service calls.\n \"\"\"\n # 1. Get a math problem - Using actual DatasetActor API\n sample = await services[\"dataloader\"].sample.call_one()\n prompt, target = sample[\"request\"], sample[\"target\"]\n\n # 2. Student generates answer - Using actual Policy API\n responses = await services[\"policy\"].generate.route(prompt=prompt)\n answer = responses[0].text\n\n # 3. Teacher grades it - Using actual RewardActor API\n score = await services[\"reward_actor\"].evaluate_response.route(\n prompt=prompt, response=answer, target=target\n )\n\n # 4. Compare to baseline - Using actual ReferenceModel API\n # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs\n input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids])\n ref_logprobs = await services[\"ref_model\"].forward.route(\n input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True\n )\n\n # 5. Store experience - Using actual Episode structure from apps/grpo/main.py\n episode = create_episode_from_response(responses[0], score, ref_logprobs, step)\n await services[\"replay_buffer\"].add.call_one(episode)\n\n # 6. Improve student - Using actual trainer pattern\n batch = await services[\"replay_buffer\"].sample.call_one(curr_policy_version=step)\n if batch is not None:\n inputs, targets = batch # GRPO returns (inputs, targets) tuple\n loss = await services[\"trainer\"].train_step.call(inputs, targets)\n\n # 7. Policy synchronization - Using actual weight update pattern\n await services[\"trainer\"].push_weights.call(step + 1)\n await services[\"policy\"].update_weights.fanout(step + 1)\n\n return loss\n\n return None\n\n\ndef create_episode_from_response(response, score, ref_logprobs, step):\n \"\"\"Helper function to create episode from response data\"\"\"\n return {\n \"response\": response,\n \"score\": score,\n \"ref_logprobs\": ref_logprobs,\n \"step\": step,\n }" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setting Up Forge Services\n\nHere's how to initialize the complete RL system with proper resource allocation.\nEach service can scale independently based on its computational needs:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "async def setup_forge_rl_system():\n \"\"\"\n Complete setup of Forge RL services with proper resource allocation.\n This example uses Qwen 3.1-1.7B model for demonstration.\n \"\"\"\n # Note: In actual Forge environment, imports would be:\n # from forge.actors.policy import Policy\n # from forge.actors.replay_buffer import ReplayBuffer\n # from forge.actors.reference_model import ReferenceModel\n # from forge.actors.trainer import RLTrainer\n # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages\n # from forge.data.rewards import MathReward, ThinkingReward\n\n model = \"Qwen/Qwen3-1.7B\"\n group_size = 1\n\n # Initialize all services with appropriate resource allocation\n services = await asyncio.gather(\n # Dataset actor (CPU intensive for I/O)\n create_dataset_actor(model),\n # Policy service (GPU for inference)\n create_policy_service(model, group_size),\n # Trainer actor (GPU for training)\n create_trainer_actor(model),\n # Replay buffer (CPU for memory management)\n create_replay_buffer_actor(),\n # Advantage computation (CPU)\n create_advantages_actor(),\n # Reference model (GPU for baseline)\n create_reference_model_actor(model),\n # Reward actor (CPU/small GPU for evaluation)\n create_reward_actor(),\n )\n\n service_names = [\n \"dataloader\",\n \"policy\",\n \"trainer\",\n \"replay_buffer\",\n \"compute_advantages\",\n \"ref_model\",\n \"reward_actor\",\n ]\n\n return dict(zip(service_names, services))\n\n\n# Service creation functions (would use actual Forge APIs)\nasync def create_dataset_actor(model):\n \"\"\"DatasetActor for loading training data\"\"\"\n return {\n \"name\": \"DatasetActor\",\n \"config\": {\n \"path\": \"openai/gsm8k\",\n \"revision\": \"main\",\n \"data_split\": \"train\",\n \"streaming\": True,\n \"model\": model,\n },\n \"resources\": \"CPU\",\n \"sample\": lambda: {\n \"call_one\": lambda: {\"request\": \"What is 2+2?\", \"target\": \"4\"}\n },\n }\n\n\nasync def create_policy_service(model, group_size):\n \"\"\"Policy service for text generation\"\"\"\n return {\n \"name\": \"Policy\",\n \"config\": {\n \"engine_config\": {\n \"model\": model,\n \"tensor_parallel_size\": 1,\n \"pipeline_parallel_size\": 1,\n \"enforce_eager\": False,\n },\n \"sampling_config\": {\n \"n\": group_size,\n \"max_tokens\": 16,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n },\n },\n \"resources\": \"GPU\",\n \"generate\": lambda: {\"route\": lambda prompt: [MockResponse()]},\n }\n\n\nasync def create_trainer_actor(model):\n \"\"\"RLTrainer for policy optimization\"\"\"\n return {\n \"name\": \"RLTrainer\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"optimizer\": {\"name\": \"AdamW\", \"lr\": 1e-5},\n \"training\": {\"local_batch_size\": 2, \"seq_len\": 2048},\n },\n \"resources\": \"GPU\",\n \"train_step\": lambda: {\"call\": lambda inputs, targets: 0.5},\n }\n\n\nasync def create_replay_buffer_actor():\n \"\"\"ReplayBuffer for experience storage\"\"\"\n return {\n \"name\": \"ReplayBuffer\",\n \"config\": {\"batch_size\": 2, \"max_policy_age\": 1, \"dp_size\": 1},\n \"resources\": \"CPU\",\n \"add\": lambda: {\"call_one\": lambda episode: None},\n \"sample\": lambda: {\"call_one\": lambda curr_policy_version: ([], [])},\n }\n\n\nasync def create_advantages_actor():\n \"\"\"ComputeAdvantages for advantage estimation\"\"\"\n return {\"name\": \"ComputeAdvantages\", \"resources\": \"CPU\"}\n\n\nasync def create_reference_model_actor(model):\n \"\"\"ReferenceModel for baseline computation\"\"\"\n return {\n \"name\": \"ReferenceModel\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"training\": {\"dtype\": \"bfloat16\"},\n },\n \"resources\": \"GPU\",\n \"forward\": lambda: {\n \"route\": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor(\n [0.1, 0.2]\n )\n },\n }\n\n\nasync def create_reward_actor():\n \"\"\"RewardActor for response evaluation\"\"\"\n return {\n \"name\": \"RewardActor\",\n \"config\": {\"reward_functions\": [\"MathReward\", \"ThinkingReward\"]},\n \"resources\": \"CPU\",\n \"evaluate_response\": lambda: {\"route\": lambda prompt, response, target: 0.95},\n }\n\n\nclass MockResponse:\n \"\"\"Mock response object for demonstration\"\"\"\n\n def __init__(self):\n self.text = \"The answer is 4\"\n self.prompt_ids = torch.tensor([1, 2, 3])\n self.token_ids = torch.tensor([4, 5, 6])\n\n\n# Demonstrate the setup\nprint(\"Setting up Forge RL system...\")\n# services = await setup_forge_rl_system() # Would work in async context\nprint(\"Forge services configured with independent scaling capabilities\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Why Forge Architecture Matters\n\nTraditional ML infrastructure fails for RL because each component has\ndifferent resource needs, scaling patterns, and failure modes:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def show_infrastructure_challenges():\n \"\"\"\n Demonstrate why traditional monolithic RL fails and how Forge solves it.\n \"\"\"\n print(\"=== Infrastructure Challenges ===\\n\")\n\n print(\"Problem 1: Different Resource Needs\")\n resource_requirements = {\n \"Policy (Student AI)\": {\n \"generates\": \"'The answer is 4'\",\n \"needs\": \"Large GPU memory\",\n \"scaling\": \"Multiple replicas for speed\",\n },\n \"Reward Model (Teacher)\": {\n \"scores\": \"answers: 0.95\",\n \"needs\": \"Moderate compute\",\n \"scaling\": \"CPU or small GPU\",\n },\n \"Trainer (Tutor)\": {\n \"improves\": \"student weights\",\n \"needs\": \"Massive GPU compute\",\n \"scaling\": \"Distributed training\",\n },\n \"Dataset (Question Bank)\": {\n \"provides\": \"'What is 2+2?'\",\n \"needs\": \"CPU intensive I/O\",\n \"scaling\": \"High memory bandwidth\",\n },\n }\n\n for component, reqs in resource_requirements.items():\n print(f\"{component}:\")\n for key, value in reqs.items():\n print(f\" {key}: {value}\")\n print()\n\n print(\"Problem 2: Coordination Complexity\")\n print(\"Unlike supervised learning with independent batches,\")\n print(\"RL requires complex coordination between components:\")\n print(\"- Policy waits idle while reward model works\")\n print(\"- Training waits for single episode (batch size = 1)\")\n print(\"- Everything stops if any component fails\")\n print()\n\n print(\"=== Forge Solutions ===\\n\")\n\n print(\"\u2705 Automatic Resource Management\")\n print(\"- Routing to least loaded replica\")\n print(\"- GPU memory management\")\n print(\"- Batch optimization\")\n print(\"- Failure recovery\")\n print(\"- Auto-scaling based on demand\")\n print()\n\n print(\"\u2705 Independent Scaling\")\n print(\"- Policy: num_replicas=8 for high inference demand\")\n print(\"- RewardActor: num_replicas=16 for parallel evaluation\")\n print(\"- Trainer: Multiple actors for distributed training\")\n print()\n\n print(\"\u2705 Fault Tolerance\")\n print(\"- Automatic routing to healthy replicas\")\n print(\"- Background replica respawn\")\n print(\"- Graceful degradation\")\n print(\"- System continues during component failures\")\n\n\nshow_infrastructure_challenges()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Production Scaling Example\n\nHere's how you would scale the system for production workloads:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def demonstrate_production_scaling():\n \"\"\"\n Show how Forge services scale independently for production.\n \"\"\"\n print(\"=== Production Scaling Configuration ===\\n\")\n\n scaling_config = {\n \"Policy Service\": {\n \"replicas\": 8,\n \"reason\": \"High inference demand from multiple training runs\",\n \"resources\": \"GPU-heavy instances\",\n },\n \"RewardActor Service\": {\n \"replicas\": 16,\n \"reason\": \"Parallel evaluation of many responses\",\n \"resources\": \"CPU/small GPU instances\",\n },\n \"Trainer Actor\": {\n \"replicas\": 4,\n \"reason\": \"Distributed training across multiple nodes\",\n \"resources\": \"Large GPU clusters\",\n },\n \"Dataset Actor\": {\n \"replicas\": 2,\n \"reason\": \"I/O intensive data loading\",\n \"resources\": \"High-bandwidth CPU instances\",\n },\n \"ReplayBuffer Actor\": {\n \"replicas\": 1,\n \"reason\": \"Centralized experience storage\",\n \"resources\": \"High-memory instances\",\n },\n }\n\n for service, config in scaling_config.items():\n print(f\"{service}:\")\n print(f\" Replicas: {config['replicas']}\")\n print(f\" Reason: {config['reason']}\")\n print(f\" Resources: {config['resources']}\")\n print()\n\n print(\"Key Benefits:\")\n print(\"- Each service scales based on its bottlenecks\")\n print(\"- Resource utilization is optimized\")\n print(\"- Costs are minimized (no idle GPUs)\")\n print(\"- System maintains performance under load\")\n\n\ndemonstrate_production_scaling()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Complete RL Training Loop\n\nHere's a complete example showing multiple RL training steps:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "async def complete_rl_training_example(num_steps: int = 5):\n \"\"\"\n Complete RL training loop using Forge services.\n \"\"\"\n print(f\"=== Running {num_steps} RL Training Steps ===\\n\")\n\n # Setup services (mock for demonstration)\n services = {\n \"dataloader\": await create_dataset_actor(\"Qwen/Qwen3-1.7B\"),\n \"policy\": await create_policy_service(\"Qwen/Qwen3-1.7B\", 1),\n \"trainer\": await create_trainer_actor(\"Qwen/Qwen3-1.7B\"),\n \"replay_buffer\": await create_replay_buffer_actor(),\n \"ref_model\": await create_reference_model_actor(\"Qwen/Qwen3-1.7B\"),\n \"reward_actor\": await create_reward_actor(),\n }\n\n losses = []\n\n for step in range(num_steps):\n print(f\"Step {step + 1}:\")\n\n # Simulate the RL step (would use actual forge_rl_step in practice)\n sample = await services[\"dataloader\"][\"sample\"]()[\"call_one\"]()\n print(f\" Question: {sample['request']}\")\n print(f\" Target: {sample['target']}\")\n\n # Generate response\n responses = await services[\"policy\"][\"generate\"]()[\"route\"](sample[\"request\"])\n print(f\" Generated: {responses[0].text}\")\n\n # Get reward\n score = await services[\"reward_actor\"][\"evaluate_response\"]()[\"route\"](\n sample[\"request\"], responses[0].text, sample[\"target\"]\n )\n print(f\" Reward: {score}\")\n\n # Simulate training (every few steps when buffer has enough data)\n if step >= 2: # Start training after accumulating some experience\n loss = await services[\"trainer\"][\"train_step\"]()[\"call\"]([], [])\n losses.append(loss)\n print(f\" Training Loss: {loss:.4f}\")\n\n print()\n\n print(f\"Training completed! Average loss: {sum(losses)/len(losses):.4f}\")\n return losses\n\n\n# Run the example (would work in async context)\nprint(\"Complete RL training example:\")\nprint(\"(In real usage, run: await complete_rl_training_example(5))\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conclusion\n\nThis tutorial demonstrated how RL fundamentals map to Forge's distributed\nservice architecture. Key takeaways:\n\n1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.)\n becomes an independent, scalable Forge service\n\n2. **Resource Optimization**: Services scale independently based on their\n computational needs (GPU for inference/training, CPU for data/rewards)\n\n3. **Fault Tolerance**: Individual service failures don't stop the entire\n training pipeline - Forge handles routing and recovery automatically\n\n4. **Simple Interface**: Complex distributed systems are hidden behind\n simple async function calls\n\nThe same RL logic that works conceptually scales to production workloads\nwithout infrastructure code - Forge handles distribution, scaling, and\nfault tolerance automatically.\n\n## Further Reading\n\n* [Forge Architecture Documentation](#)\n* [GRPO Implementation (apps/grpo/main.py)](#)\n* [Forge Service APIs](#)\n* [Production RL Scaling Guide](#)\n\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.18" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py deleted file mode 100644 index 3e77639a1..000000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py +++ /dev/null @@ -1,558 +0,0 @@ -""" -RL Fundamentals Using Forge Terminology -======================================== - -**Author:** `Your Name `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) - * How RL concepts map to distributed Forge services - * Building scalable RL training loops with fault tolerance - * Resource management and independent scaling patterns - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0+ - * GPU access recommended - * Basic understanding of reinforcement learning - * Familiarity with async/await in Python - -This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. -We'll start with a simple math tutoring example to understand how traditional RL concepts -map to Forge's distributed service model. - -""" - -######################################################################## -# Core RL Components in Forge -# ---------------------------- -# -# Let's start with a simple math tutoring example to understand RL concepts -# with the exact names Forge uses. Think of it as teaching an AI student: -# -# - **Dataset**: Provides questions (like "What is 2+2?") -# - **Policy**: The AI student being trained (generates answers like "The answer is 4") -# - **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) -# - **Reference Model**: Copy of original student (prevents drift from baseline) -# - **Replay Buffer**: Notebook that stores experiences (question + answer + score) -# - **Trainer**: The tutor that improves the student based on experiences -# -# Here's how these components interact in a conceptual RL step: - -import asyncio -from typing import Any, Dict, Optional - -import torch - - -def conceptual_rl_step(): - """ - Conceptual example showing the RL learning flow. - See apps/grpo/main.py for actual GRPO implementation. - """ - # 1. Get a math problem - question = "What is 2+2?" # dataset.sample() - - # 2. Student generates answer - answer = "The answer is 4" # policy.generate(question) - - # 3. Teacher grades it - score = 0.95 # reward_model.evaluate(question, answer) - - # 4. Compare to original student - baseline = 0.85 # reference_model.compute_logprobs(question, answer) - - # 5. Store the experience - experience = { - "question": question, - "answer": answer, - "score": score, - "baseline": baseline, - } - # replay_buffer.add(experience) - - # 6. When enough experiences collected, improve student - # trainer.train_step(batch) # Student gets better! - - return experience - - -example_experience = conceptual_rl_step() -print("Example RL experience:", example_experience) - -######################################################################## -# From Concepts to Forge Services -# -------------------------------- -# -# Here's the key insight: **Each RL component becomes a Forge service**. -# The toy example above maps directly to Forge's distributed architecture: -# -# * Dataset → DatasetActor -# * Policy → Policy -# * Reward Model → RewardActor -# * Reference Model → ReferenceModel -# * Replay Buffer → ReplayBuffer -# * Trainer → RLTrainer -# -# Let's see how the conceptual example translates to actual Forge service calls: - - -async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: - """ - RL step using actual Forge service APIs. - This shows the same logic as conceptual_rl_step but with real service calls. - """ - # 1. Get a math problem - Using actual DatasetActor API - sample = await services["dataloader"].sample.call_one() - prompt, target = sample["request"], sample["target"] - - # 2. Student generates answer - Using actual Policy API - responses = await services["policy"].generate.route(prompt=prompt) - answer = responses[0].text - - # 3. Teacher grades it - Using actual RewardActor API - score = await services["reward_actor"].evaluate_response.route( - prompt=prompt, response=answer, target=target - ) - - # 4. Compare to baseline - Using actual ReferenceModel API - # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs - input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) - ref_logprobs = await services["ref_model"].forward.route( - input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True - ) - - # 5. Store experience - Using actual Episode structure from apps/grpo/main.py - episode = create_episode_from_response(responses[0], score, ref_logprobs, step) - await services["replay_buffer"].add.call_one(episode) - - # 6. Improve student - Using actual trainer pattern - batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) - if batch is not None: - inputs, targets = batch # GRPO returns (inputs, targets) tuple - loss = await services["trainer"].train_step.call(inputs, targets) - - # 7. Policy synchronization - Using actual weight update pattern - await services["trainer"].push_weights.call(step + 1) - await services["policy"].update_weights.fanout(step + 1) - - return loss - - return None - - -def create_episode_from_response(response, score, ref_logprobs, step): - """Helper function to create episode from response data""" - return { - "response": response, - "score": score, - "ref_logprobs": ref_logprobs, - "step": step, - } - - -######################################################################## -# Setting Up Forge Services -# -------------------------- -# -# Here's how to initialize the complete RL system with proper resource allocation. -# Each service can scale independently based on its computational needs: - - -async def setup_forge_rl_system(): - """ - Complete setup of Forge RL services with proper resource allocation. - This example uses Qwen 3.1-1.7B model for demonstration. - """ - # Note: In actual Forge environment, imports would be: - # from forge.actors.policy import Policy - # from forge.actors.replay_buffer import ReplayBuffer - # from forge.actors.reference_model import ReferenceModel - # from forge.actors.trainer import RLTrainer - # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages - # from forge.data.rewards import MathReward, ThinkingReward - - model = "Qwen/Qwen3-1.7B" - group_size = 1 - - # Initialize all services with appropriate resource allocation - services = await asyncio.gather( - # Dataset actor (CPU intensive for I/O) - create_dataset_actor(model), - # Policy service (GPU for inference) - create_policy_service(model, group_size), - # Trainer actor (GPU for training) - create_trainer_actor(model), - # Replay buffer (CPU for memory management) - create_replay_buffer_actor(), - # Advantage computation (CPU) - create_advantages_actor(), - # Reference model (GPU for baseline) - create_reference_model_actor(model), - # Reward actor (CPU/small GPU for evaluation) - create_reward_actor(), - ) - - service_names = [ - "dataloader", - "policy", - "trainer", - "replay_buffer", - "compute_advantages", - "ref_model", - "reward_actor", - ] - - return dict(zip(service_names, services)) - - -# Service creation functions (would use actual Forge APIs) -async def create_dataset_actor(model): - """DatasetActor for loading training data""" - return { - "name": "DatasetActor", - "config": { - "path": "openai/gsm8k", - "revision": "main", - "data_split": "train", - "streaming": True, - "model": model, - }, - "resources": "CPU", - "sample": lambda: { - "call_one": lambda: {"request": "What is 2+2?", "target": "4"} - }, - } - - -async def create_policy_service(model, group_size): - """Policy service for text generation""" - return { - "name": "Policy", - "config": { - "engine_config": { - "model": model, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "enforce_eager": False, - }, - "sampling_config": { - "n": group_size, - "max_tokens": 16, - "temperature": 1.0, - "top_p": 1.0, - }, - }, - "resources": "GPU", - "generate": lambda: {"route": lambda prompt: [MockResponse()]}, - } - - -async def create_trainer_actor(model): - """RLTrainer for policy optimization""" - return { - "name": "RLTrainer", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "optimizer": {"name": "AdamW", "lr": 1e-5}, - "training": {"local_batch_size": 2, "seq_len": 2048}, - }, - "resources": "GPU", - "train_step": lambda: {"call": lambda inputs, targets: 0.5}, - } - - -async def create_replay_buffer_actor(): - """ReplayBuffer for experience storage""" - return { - "name": "ReplayBuffer", - "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, - "resources": "CPU", - "add": lambda: {"call_one": lambda episode: None}, - "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, - } - - -async def create_advantages_actor(): - """ComputeAdvantages for advantage estimation""" - return {"name": "ComputeAdvantages", "resources": "CPU"} - - -async def create_reference_model_actor(model): - """ReferenceModel for baseline computation""" - return { - "name": "ReferenceModel", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "training": {"dtype": "bfloat16"}, - }, - "resources": "GPU", - "forward": lambda: { - "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( - [0.1, 0.2] - ) - }, - } - - -async def create_reward_actor(): - """RewardActor for response evaluation""" - return { - "name": "RewardActor", - "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, - "resources": "CPU", - "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, - } - - -class MockResponse: - """Mock response object for demonstration""" - - def __init__(self): - self.text = "The answer is 4" - self.prompt_ids = torch.tensor([1, 2, 3]) - self.token_ids = torch.tensor([4, 5, 6]) - - -# Demonstrate the setup -print("Setting up Forge RL system...") -# services = await setup_forge_rl_system() # Would work in async context -print("Forge services configured with independent scaling capabilities") - -######################################################################## -# Why Forge Architecture Matters -# ------------------------------- -# -# Traditional ML infrastructure fails for RL because each component has -# different resource needs, scaling patterns, and failure modes: - - -def show_infrastructure_challenges(): - """ - Demonstrate why traditional monolithic RL fails and how Forge solves it. - """ - print("=== Infrastructure Challenges ===\n") - - print("Problem 1: Different Resource Needs") - resource_requirements = { - "Policy (Student AI)": { - "generates": "'The answer is 4'", - "needs": "Large GPU memory", - "scaling": "Multiple replicas for speed", - }, - "Reward Model (Teacher)": { - "scores": "answers: 0.95", - "needs": "Moderate compute", - "scaling": "CPU or small GPU", - }, - "Trainer (Tutor)": { - "improves": "student weights", - "needs": "Massive GPU compute", - "scaling": "Distributed training", - }, - "Dataset (Question Bank)": { - "provides": "'What is 2+2?'", - "needs": "CPU intensive I/O", - "scaling": "High memory bandwidth", - }, - } - - for component, reqs in resource_requirements.items(): - print(f"{component}:") - for key, value in reqs.items(): - print(f" {key}: {value}") - print() - - print("Problem 2: Coordination Complexity") - print("Unlike supervised learning with independent batches,") - print("RL requires complex coordination between components:") - print("- Policy waits idle while reward model works") - print("- Training waits for single episode (batch size = 1)") - print("- Everything stops if any component fails") - print() - - print("=== Forge Solutions ===\n") - - print("✅ Automatic Resource Management") - print("- Routing to least loaded replica") - print("- GPU memory management") - print("- Batch optimization") - print("- Failure recovery") - print("- Auto-scaling based on demand") - print() - - print("✅ Independent Scaling") - print("- Policy: num_replicas=8 for high inference demand") - print("- RewardActor: num_replicas=16 for parallel evaluation") - print("- Trainer: Multiple actors for distributed training") - print() - - print("✅ Fault Tolerance") - print("- Automatic routing to healthy replicas") - print("- Background replica respawn") - print("- Graceful degradation") - print("- System continues during component failures") - - -show_infrastructure_challenges() - -######################################################################## -# Production Scaling Example -# --------------------------- -# -# Here's how you would scale the system for production workloads: - - -def demonstrate_production_scaling(): - """ - Show how Forge services scale independently for production. - """ - print("=== Production Scaling Configuration ===\n") - - scaling_config = { - "Policy Service": { - "replicas": 8, - "reason": "High inference demand from multiple training runs", - "resources": "GPU-heavy instances", - }, - "RewardActor Service": { - "replicas": 16, - "reason": "Parallel evaluation of many responses", - "resources": "CPU/small GPU instances", - }, - "Trainer Actor": { - "replicas": 4, - "reason": "Distributed training across multiple nodes", - "resources": "Large GPU clusters", - }, - "Dataset Actor": { - "replicas": 2, - "reason": "I/O intensive data loading", - "resources": "High-bandwidth CPU instances", - }, - "ReplayBuffer Actor": { - "replicas": 1, - "reason": "Centralized experience storage", - "resources": "High-memory instances", - }, - } - - for service, config in scaling_config.items(): - print(f"{service}:") - print(f" Replicas: {config['replicas']}") - print(f" Reason: {config['reason']}") - print(f" Resources: {config['resources']}") - print() - - print("Key Benefits:") - print("- Each service scales based on its bottlenecks") - print("- Resource utilization is optimized") - print("- Costs are minimized (no idle GPUs)") - print("- System maintains performance under load") - - -demonstrate_production_scaling() - -######################################################################## -# Complete RL Training Loop -# -------------------------- -# -# Here's a complete example showing multiple RL training steps: - - -async def complete_rl_training_example(num_steps: int = 5): - """ - Complete RL training loop using Forge services. - """ - print(f"=== Running {num_steps} RL Training Steps ===\n") - - # Setup services (mock for demonstration) - services = { - "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), - "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), - "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), - "replay_buffer": await create_replay_buffer_actor(), - "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), - "reward_actor": await create_reward_actor(), - } - - losses = [] - - for step in range(num_steps): - print(f"Step {step + 1}:") - - # Simulate the RL step (would use actual forge_rl_step in practice) - sample = await services["dataloader"]["sample"]()["call_one"]() - print(f" Question: {sample['request']}") - print(f" Target: {sample['target']}") - - # Generate response - responses = await services["policy"]["generate"]()["route"](sample["request"]) - print(f" Generated: {responses[0].text}") - - # Get reward - score = await services["reward_actor"]["evaluate_response"]()["route"]( - sample["request"], responses[0].text, sample["target"] - ) - print(f" Reward: {score}") - - # Simulate training (every few steps when buffer has enough data) - if step >= 2: # Start training after accumulating some experience - loss = await services["trainer"]["train_step"]()["call"]([], []) - losses.append(loss) - print(f" Training Loss: {loss:.4f}") - - print() - - print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") - return losses - - -# Run the example (would work in async context) -print("Complete RL training example:") -print("(In real usage, run: await complete_rl_training_example(5))") - -######################################################################## -# Conclusion -# ---------- -# -# This tutorial demonstrated how RL fundamentals map to Forge's distributed -# service architecture. Key takeaways: -# -# 1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) -# becomes an independent, scalable Forge service -# -# 2. **Resource Optimization**: Services scale independently based on their -# computational needs (GPU for inference/training, CPU for data/rewards) -# -# 3. **Fault Tolerance**: Individual service failures don't stop the entire -# training pipeline - Forge handles routing and recovery automatically -# -# 4. **Simple Interface**: Complex distributed systems are hidden behind -# simple async function calls -# -# The same RL logic that works conceptually scales to production workloads -# without infrastructure code - Forge handles distribution, scaling, and -# fault tolerance automatically. -# -# Further Reading -# --------------- -# -# * `Forge Architecture Documentation <#>`_ -# * `GRPO Implementation (apps/grpo/main.py) <#>`_ -# * `Forge Service APIs <#>`_ -# * `Production RL Scaling Guide <#>`_ - diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 deleted file mode 100644 index d93f4a3ab..000000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 +++ /dev/null @@ -1 +0,0 @@ -d1dbe1df9283c920a3039b06b0d5ce4d \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst b/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst deleted file mode 100644 index 95ad7c0ba..000000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst +++ /dev/null @@ -1,788 +0,0 @@ - -.. DO NOT EDIT. -.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. -.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: -.. "tutorials/rl_fundamentals_forge_tutorial.py" -.. LINE NUMBERS ARE GIVEN BELOW. - -.. only:: html - - .. note:: - :class: sphx-glr-download-link-note - - :ref:`Go to the end ` - to download the full example code. - -.. rst-class:: sphx-glr-example-title - -.. _sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py: - - -RL Fundamentals Using Forge Terminology -======================================== - -**Author:** `Your Name `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) - * How RL concepts map to distributed Forge services - * Building scalable RL training loops with fault tolerance - * Resource management and independent scaling patterns - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0+ - * GPU access recommended - * Basic understanding of reinforcement learning - * Familiarity with async/await in Python - -This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. -We'll start with a simple math tutoring example to understand how traditional RL concepts -map to Forge's distributed service model. - -.. GENERATED FROM PYTHON SOURCE LINES 32-46 - -Core RL Components in Forge ----------------------------- - -Let's start with a simple math tutoring example to understand RL concepts -with the exact names Forge uses. Think of it as teaching an AI student: - -- **Dataset**: Provides questions (like "What is 2+2?") -- **Policy**: The AI student being trained (generates answers like "The answer is 4") -- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) -- **Reference Model**: Copy of original student (prevents drift from baseline) -- **Replay Buffer**: Notebook that stores experiences (question + answer + score) -- **Trainer**: The tutor that improves the student based on experiences - -Here's how these components interact in a conceptual RL step: - -.. GENERATED FROM PYTHON SOURCE LINES 46-88 - -.. code-block:: Python - - - import asyncio - from typing import Any, Dict, Optional - - import torch - - - def conceptual_rl_step(): - """ - Conceptual example showing the RL learning flow. - See apps/grpo/main.py for actual GRPO implementation. - """ - # 1. Get a math problem - question = "What is 2+2?" # dataset.sample() - - # 2. Student generates answer - answer = "The answer is 4" # policy.generate(question) - - # 3. Teacher grades it - score = 0.95 # reward_model.evaluate(question, answer) - - # 4. Compare to original student - baseline = 0.85 # reference_model.compute_logprobs(question, answer) - - # 5. Store the experience - experience = { - "question": question, - "answer": answer, - "score": score, - "baseline": baseline, - } - # replay_buffer.add(experience) - - # 6. When enough experiences collected, improve student - # trainer.train_step(batch) # Student gets better! - - return experience - - - example_experience = conceptual_rl_step() - print("Example RL experience:", example_experience) - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - Example RL experience: {'question': 'What is 2+2?', 'answer': 'The answer is 4', 'score': 0.95, 'baseline': 0.85} - - - - -.. GENERATED FROM PYTHON SOURCE LINES 89-103 - -From Concepts to Forge Services --------------------------------- - -Here's the key insight: **Each RL component becomes a Forge service**. -The toy example above maps directly to Forge's distributed architecture: - -* Dataset → DatasetActor -* Policy → Policy -* Reward Model → RewardActor -* Reference Model → ReferenceModel -* Replay Buffer → ReplayBuffer -* Trainer → RLTrainer - -Let's see how the conceptual example translates to actual Forge service calls: - -.. GENERATED FROM PYTHON SOURCE LINES 103-159 - -.. code-block:: Python - - - - async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: - """ - RL step using actual Forge service APIs. - This shows the same logic as conceptual_rl_step but with real service calls. - """ - # 1. Get a math problem - Using actual DatasetActor API - sample = await services["dataloader"].sample.call_one() - prompt, target = sample["request"], sample["target"] - - # 2. Student generates answer - Using actual Policy API - responses = await services["policy"].generate.route(prompt=prompt) - answer = responses[0].text - - # 3. Teacher grades it - Using actual RewardActor API - score = await services["reward_actor"].evaluate_response.route( - prompt=prompt, response=answer, target=target - ) - - # 4. Compare to baseline - Using actual ReferenceModel API - # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs - input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) - ref_logprobs = await services["ref_model"].forward.route( - input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True - ) - - # 5. Store experience - Using actual Episode structure from apps/grpo/main.py - episode = create_episode_from_response(responses[0], score, ref_logprobs, step) - await services["replay_buffer"].add.call_one(episode) - - # 6. Improve student - Using actual trainer pattern - batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) - if batch is not None: - inputs, targets = batch # GRPO returns (inputs, targets) tuple - loss = await services["trainer"].train_step.call(inputs, targets) - - # 7. Policy synchronization - Using actual weight update pattern - await services["trainer"].push_weights.call(step + 1) - await services["policy"].update_weights.fanout(step + 1) - - return loss - - return None - - - def create_episode_from_response(response, score, ref_logprobs, step): - """Helper function to create episode from response data""" - return { - "response": response, - "score": score, - "ref_logprobs": ref_logprobs, - "step": step, - } - - - - - - - - - -.. GENERATED FROM PYTHON SOURCE LINES 160-165 - -Setting Up Forge Services --------------------------- - -Here's how to initialize the complete RL system with proper resource allocation. -Each service can scale independently based on its computational needs: - -.. GENERATED FROM PYTHON SOURCE LINES 165-335 - -.. code-block:: Python - - - - async def setup_forge_rl_system(): - """ - Complete setup of Forge RL services with proper resource allocation. - This example uses Qwen 3.1-1.7B model for demonstration. - """ - # Note: In actual Forge environment, imports would be: - # from forge.actors.policy import Policy - # from forge.actors.replay_buffer import ReplayBuffer - # from forge.actors.reference_model import ReferenceModel - # from forge.actors.trainer import RLTrainer - # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages - # from forge.data.rewards import MathReward, ThinkingReward - - model = "Qwen/Qwen3-1.7B" - group_size = 1 - - # Initialize all services with appropriate resource allocation - services = await asyncio.gather( - # Dataset actor (CPU intensive for I/O) - create_dataset_actor(model), - # Policy service (GPU for inference) - create_policy_service(model, group_size), - # Trainer actor (GPU for training) - create_trainer_actor(model), - # Replay buffer (CPU for memory management) - create_replay_buffer_actor(), - # Advantage computation (CPU) - create_advantages_actor(), - # Reference model (GPU for baseline) - create_reference_model_actor(model), - # Reward actor (CPU/small GPU for evaluation) - create_reward_actor(), - ) - - service_names = [ - "dataloader", - "policy", - "trainer", - "replay_buffer", - "compute_advantages", - "ref_model", - "reward_actor", - ] - - return dict(zip(service_names, services)) - - - # Service creation functions (would use actual Forge APIs) - async def create_dataset_actor(model): - """DatasetActor for loading training data""" - return { - "name": "DatasetActor", - "config": { - "path": "openai/gsm8k", - "revision": "main", - "data_split": "train", - "streaming": True, - "model": model, - }, - "resources": "CPU", - "sample": lambda: { - "call_one": lambda: {"request": "What is 2+2?", "target": "4"} - }, - } - - - async def create_policy_service(model, group_size): - """Policy service for text generation""" - return { - "name": "Policy", - "config": { - "engine_config": { - "model": model, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "enforce_eager": False, - }, - "sampling_config": { - "n": group_size, - "max_tokens": 16, - "temperature": 1.0, - "top_p": 1.0, - }, - }, - "resources": "GPU", - "generate": lambda: {"route": lambda prompt: [MockResponse()]}, - } - - - async def create_trainer_actor(model): - """RLTrainer for policy optimization""" - return { - "name": "RLTrainer", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "optimizer": {"name": "AdamW", "lr": 1e-5}, - "training": {"local_batch_size": 2, "seq_len": 2048}, - }, - "resources": "GPU", - "train_step": lambda: {"call": lambda inputs, targets: 0.5}, - } - - - async def create_replay_buffer_actor(): - """ReplayBuffer for experience storage""" - return { - "name": "ReplayBuffer", - "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, - "resources": "CPU", - "add": lambda: {"call_one": lambda episode: None}, - "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, - } - - - async def create_advantages_actor(): - """ComputeAdvantages for advantage estimation""" - return {"name": "ComputeAdvantages", "resources": "CPU"} - - - async def create_reference_model_actor(model): - """ReferenceModel for baseline computation""" - return { - "name": "ReferenceModel", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "training": {"dtype": "bfloat16"}, - }, - "resources": "GPU", - "forward": lambda: { - "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( - [0.1, 0.2] - ) - }, - } - - - async def create_reward_actor(): - """RewardActor for response evaluation""" - return { - "name": "RewardActor", - "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, - "resources": "CPU", - "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, - } - - - class MockResponse: - """Mock response object for demonstration""" - - def __init__(self): - self.text = "The answer is 4" - self.prompt_ids = torch.tensor([1, 2, 3]) - self.token_ids = torch.tensor([4, 5, 6]) - - - # Demonstrate the setup - print("Setting up Forge RL system...") - # services = await setup_forge_rl_system() # Would work in async context - print("Forge services configured with independent scaling capabilities") - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - Setting up Forge RL system... - Forge services configured with independent scaling capabilities - - - - -.. GENERATED FROM PYTHON SOURCE LINES 336-341 - -Why Forge Architecture Matters -------------------------------- - -Traditional ML infrastructure fails for RL because each component has -different resource needs, scaling patterns, and failure modes: - -.. GENERATED FROM PYTHON SOURCE LINES 341-412 - -.. code-block:: Python - - - - def show_infrastructure_challenges(): - """ - Demonstrate why traditional monolithic RL fails and how Forge solves it. - """ - print("=== Infrastructure Challenges ===\n") - - print("Problem 1: Different Resource Needs") - resource_requirements = { - "Policy (Student AI)": { - "generates": "'The answer is 4'", - "needs": "Large GPU memory", - "scaling": "Multiple replicas for speed", - }, - "Reward Model (Teacher)": { - "scores": "answers: 0.95", - "needs": "Moderate compute", - "scaling": "CPU or small GPU", - }, - "Trainer (Tutor)": { - "improves": "student weights", - "needs": "Massive GPU compute", - "scaling": "Distributed training", - }, - "Dataset (Question Bank)": { - "provides": "'What is 2+2?'", - "needs": "CPU intensive I/O", - "scaling": "High memory bandwidth", - }, - } - - for component, reqs in resource_requirements.items(): - print(f"{component}:") - for key, value in reqs.items(): - print(f" {key}: {value}") - print() - - print("Problem 2: Coordination Complexity") - print("Unlike supervised learning with independent batches,") - print("RL requires complex coordination between components:") - print("- Policy waits idle while reward model works") - print("- Training waits for single episode (batch size = 1)") - print("- Everything stops if any component fails") - print() - - print("=== Forge Solutions ===\n") - - print("✅ Automatic Resource Management") - print("- Routing to least loaded replica") - print("- GPU memory management") - print("- Batch optimization") - print("- Failure recovery") - print("- Auto-scaling based on demand") - print() - - print("✅ Independent Scaling") - print("- Policy: num_replicas=8 for high inference demand") - print("- RewardActor: num_replicas=16 for parallel evaluation") - print("- Trainer: Multiple actors for distributed training") - print() - - print("✅ Fault Tolerance") - print("- Automatic routing to healthy replicas") - print("- Background replica respawn") - print("- Graceful degradation") - print("- System continues during component failures") - - - show_infrastructure_challenges() - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - === Infrastructure Challenges === - - Problem 1: Different Resource Needs - Policy (Student AI): - generates: 'The answer is 4' - needs: Large GPU memory - scaling: Multiple replicas for speed - - Reward Model (Teacher): - scores: answers: 0.95 - needs: Moderate compute - scaling: CPU or small GPU - - Trainer (Tutor): - improves: student weights - needs: Massive GPU compute - scaling: Distributed training - - Dataset (Question Bank): - provides: 'What is 2+2?' - needs: CPU intensive I/O - scaling: High memory bandwidth - - Problem 2: Coordination Complexity - Unlike supervised learning with independent batches, - RL requires complex coordination between components: - - Policy waits idle while reward model works - - Training waits for single episode (batch size = 1) - - Everything stops if any component fails - - === Forge Solutions === - - ✅ Automatic Resource Management - - Routing to least loaded replica - - GPU memory management - - Batch optimization - - Failure recovery - - Auto-scaling based on demand - - ✅ Independent Scaling - - Policy: num_replicas=8 for high inference demand - - RewardActor: num_replicas=16 for parallel evaluation - - Trainer: Multiple actors for distributed training - - ✅ Fault Tolerance - - Automatic routing to healthy replicas - - Background replica respawn - - Graceful degradation - - System continues during component failures - - - - -.. GENERATED FROM PYTHON SOURCE LINES 413-417 - -Production Scaling Example ---------------------------- - -Here's how you would scale the system for production workloads: - -.. GENERATED FROM PYTHON SOURCE LINES 417-469 - -.. code-block:: Python - - - - def demonstrate_production_scaling(): - """ - Show how Forge services scale independently for production. - """ - print("=== Production Scaling Configuration ===\n") - - scaling_config = { - "Policy Service": { - "replicas": 8, - "reason": "High inference demand from multiple training runs", - "resources": "GPU-heavy instances", - }, - "RewardActor Service": { - "replicas": 16, - "reason": "Parallel evaluation of many responses", - "resources": "CPU/small GPU instances", - }, - "Trainer Actor": { - "replicas": 4, - "reason": "Distributed training across multiple nodes", - "resources": "Large GPU clusters", - }, - "Dataset Actor": { - "replicas": 2, - "reason": "I/O intensive data loading", - "resources": "High-bandwidth CPU instances", - }, - "ReplayBuffer Actor": { - "replicas": 1, - "reason": "Centralized experience storage", - "resources": "High-memory instances", - }, - } - - for service, config in scaling_config.items(): - print(f"{service}:") - print(f" Replicas: {config['replicas']}") - print(f" Reason: {config['reason']}") - print(f" Resources: {config['resources']}") - print() - - print("Key Benefits:") - print("- Each service scales based on its bottlenecks") - print("- Resource utilization is optimized") - print("- Costs are minimized (no idle GPUs)") - print("- System maintains performance under load") - - - demonstrate_production_scaling() - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - === Production Scaling Configuration === - - Policy Service: - Replicas: 8 - Reason: High inference demand from multiple training runs - Resources: GPU-heavy instances - - RewardActor Service: - Replicas: 16 - Reason: Parallel evaluation of many responses - Resources: CPU/small GPU instances - - Trainer Actor: - Replicas: 4 - Reason: Distributed training across multiple nodes - Resources: Large GPU clusters - - Dataset Actor: - Replicas: 2 - Reason: I/O intensive data loading - Resources: High-bandwidth CPU instances - - ReplayBuffer Actor: - Replicas: 1 - Reason: Centralized experience storage - Resources: High-memory instances - - Key Benefits: - - Each service scales based on its bottlenecks - - Resource utilization is optimized - - Costs are minimized (no idle GPUs) - - System maintains performance under load - - - - -.. GENERATED FROM PYTHON SOURCE LINES 470-474 - -Complete RL Training Loop --------------------------- - -Here's a complete example showing multiple RL training steps: - -.. GENERATED FROM PYTHON SOURCE LINES 474-528 - -.. code-block:: Python - - - - async def complete_rl_training_example(num_steps: int = 5): - """ - Complete RL training loop using Forge services. - """ - print(f"=== Running {num_steps} RL Training Steps ===\n") - - # Setup services (mock for demonstration) - services = { - "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), - "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), - "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), - "replay_buffer": await create_replay_buffer_actor(), - "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), - "reward_actor": await create_reward_actor(), - } - - losses = [] - - for step in range(num_steps): - print(f"Step {step + 1}:") - - # Simulate the RL step (would use actual forge_rl_step in practice) - sample = await services["dataloader"]["sample"]()["call_one"]() - print(f" Question: {sample['request']}") - print(f" Target: {sample['target']}") - - # Generate response - responses = await services["policy"]["generate"]()["route"](sample["request"]) - print(f" Generated: {responses[0].text}") - - # Get reward - score = await services["reward_actor"]["evaluate_response"]()["route"]( - sample["request"], responses[0].text, sample["target"] - ) - print(f" Reward: {score}") - - # Simulate training (every few steps when buffer has enough data) - if step >= 2: # Start training after accumulating some experience - loss = await services["trainer"]["train_step"]()["call"]([], []) - losses.append(loss) - print(f" Training Loss: {loss:.4f}") - - print() - - print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") - return losses - - - # Run the example (would work in async context) - print("Complete RL training example:") - print("(In real usage, run: await complete_rl_training_example(5))") - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - Complete RL training example: - (In real usage, run: await complete_rl_training_example(5)) - - - - -.. GENERATED FROM PYTHON SOURCE LINES 529-558 - -Conclusion ----------- - -This tutorial demonstrated how RL fundamentals map to Forge's distributed -service architecture. Key takeaways: - -1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) - becomes an independent, scalable Forge service - -2. **Resource Optimization**: Services scale independently based on their - computational needs (GPU for inference/training, CPU for data/rewards) - -3. **Fault Tolerance**: Individual service failures don't stop the entire - training pipeline - Forge handles routing and recovery automatically - -4. **Simple Interface**: Complex distributed systems are hidden behind - simple async function calls - -The same RL logic that works conceptually scales to production workloads -without infrastructure code - Forge handles distribution, scaling, and -fault tolerance automatically. - -Further Reading ---------------- - -* `Forge Architecture Documentation <#>`_ -* `GRPO Implementation (apps/grpo/main.py) <#>`_ -* `Forge Service APIs <#>`_ -* `Production RL Scaling Guide <#>`_ - - -.. rst-class:: sphx-glr-timing - - **Total running time of the script:** (0 minutes 0.006 seconds) - - -.. _sphx_glr_download_tutorials_rl_fundamentals_forge_tutorial.py: - -.. only:: html - - .. container:: sphx-glr-footer sphx-glr-footer-example - - .. container:: sphx-glr-download sphx-glr-download-jupyter - - :download:`Download Jupyter notebook: rl_fundamentals_forge_tutorial.ipynb ` - - .. container:: sphx-glr-download sphx-glr-download-python - - :download:`Download Python source code: rl_fundamentals_forge_tutorial.py ` - - .. container:: sphx-glr-download sphx-glr-download-zip - - :download:`Download zipped: rl_fundamentals_forge_tutorial.zip ` diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip b/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip deleted file mode 100644 index c77b9084e4c0471638195cc647797a00896fc0af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40960 zcmdU2&2Ah?a+ZG&ycyUBs4d!R<;m;HKC zPv@gz+?`$JdA@P@b@nW1lzmf;Cg<5RHqBm^^UKj>I-Z_iZ9HD{&&I~~_LD_@F`e&k zZ)e~A<#aL6ehn6~pI_AVtlGbS|9n(mEKa+F>E-<^JY7`fe1ZnQ_^!XP(d}mE^U-jB zKYO^bk>QW|V0ToPm%D>v&PV&xK|LBwC;Q)BPUm$o-#wk;!JqGymw*0U_U%PcXP}$C z8I8x;xGd%qAp~0cgK<$+N+lz$-PydHm)|c&706fmLbTh?o=)dw2FZZO*>nQ&RoQ4_ zgwpw`$$qvgmAeOzD4$JXQR@Q^=mVM=R_RHxTr8F6oGtjv#W?4PW zhNG&Uk4_hLIW%-3hS#G3*x<17XFL0RF&Ym!8`Yo~7pG%ksh$_32|pZ9r?U#9tS_>& zVll2kbzII1yv9>FEGtNOP~xqV;=H6HWW{8N!3@h;iT~ku5+}V`QP-FWBZk#-AA|ed z#bi1kLVi~A2azy!5I2zd!PU#@d~lJye%O7`eee@Mh_4R5$%+9aQ)TlKGlEeJ%b|b% zc~Ol98FaXuS2ae*!A#H4b~J&;4|LR`2zcVVJu5Cp<54jm)mJLTqPm(4?iX*0QO(&t zxPlT-Ha1>fjH=8onyfC1!9`hdO2f64E&Q_isLIN>#Q+oHmybxj!0@1GgL*M9yBpt@ zA3;IE@w_(g&#KYo46}S$U{3Xo7z${~r%)O{#_R%W1yvZ1>d|ykj6)UMFe)Z%7|evK zW|S1HfRAhHt*=q#->|1PumLzPj^@R+1Nybuge+?-&Q8FV0D>+maL5WqBN61 z6H}R`29r~DGfeE{6*UF(U0{kiv(%kpl0A72CKg;C`@+|5w!LkZ{r2{Lb}*m59t}b6 z`$buC0;;St9=$5FT&xfD<>5~r{zbmUA)}|-33hx5c0BFu6e@boX&g;p20G{E1Pd6J z2rN`@pv{>QWiS0C1L;SOZfJKbb=A*#R?`fYuZ!^_AP9c3)}im6^U-U(rW(M`nYTRX z{_GRW@L_p|bqz_XO(X56)7cee1=n#tqQV*ZWF6Sh*J2@u^U+zIoz170*(nSpj7sSV z&c?+R4B8n8liaVTb$L3SzEVCbcqH&qzMYlx5#I}WTG3`dv19*<@@r|n6tM|43P{$A zlEW&(1c3-=-||V8hS_uyzD4Html!IPP8Jkab{QLetYI!?*n@{hGn|bW}onX0z)4`Fu9Le+gZK=g(jnVN`35 z<*UPk-;md_SjvJNYES#x&1|omeN|$RYMLNjIHOCU?xyK+L;168GaIU>>{epII$P>t zH?xP`>`0ea*3@|+YLp!$o2pN$GjY3J+tX=^BmXhhwwUQTf{B9bbW{sn(f90eMqMYR zxj62=8biBS9K9W*c#iI)E@t3zRvbB|d4pAadC6zyC1x?1_b{e_+wUW)q9Ci~ z@B4&eVkAO6_KQPa~1Nk@j+ztqj?2r2N zy-=kLxwQMIvb4L!aM5?MJbe&Q!dUOfjnWZ5qZKvF9d>ML9M_SJ5m)#&|)*>PvZDJKen2Ez+PW%c;VaoGqt!l*~R z>2_v(Y4-1b`*-%|Cj+QD-n4DPYI&xfQ$7uyojlVIe7AW zNfYB)^x@a$w=h0|MK$gs&>klnDC(pdi`&J-86&G3DjR+w{e^QV$FTQBeG~r|8$C(m4&bi}5*p^e6;+T$e zZIhp@;h>s)tBN{~5yxQ1i1;$Wv^p*@Y+YA(K7}Vh1Vv6C>p#l8^MH=t@q?3YUB0a? zizz49qnONwct_xZTZH!n(;&9bX?0*^!x3IxMgf4uO&UlJ+NZT%&VS;J-Nx~&z zyhTs#2uW>bKkZtl%U#UHd^#EZPWAy!DZVKgKV*v;>~9&#l7yNj#WOQoR2O~iTq!%0 z7q)hLDzj8?XD#(@uKQVmAhPaw0!}6-ndG!9`={z7`s&_fsl;E zwi2#h7ei)OhinDy8j!t^-B1}ugRtE;Lzr!U41(}w!Fjlrq-U_SoK$j3ijh73v7!5& z#;QjMySWAZW_FV(HH=Rsn3`g3Ae)5*#qV^-!;R9ogq|^uxdQu_8gRf2Q}U3L<~HH> zr!dyCZQaZyu=U}{1a_xDPA$lIB2=Z%kz~LK+Xmf`2?0bhld>E(LNg?L7c-MRXMiTW zwD;zQTWQTyiH3Y+-bC%!v(I%PftcOiGgFoQ_6@d9A9we5_qsp*TvK7LP6+4l`y{nGaubK`os$US%+{aKfZkS-G;wrLo9 z+H}4IqC2wTM~wDl_`1L$%DG9;N6a%8?P~ahutU>9FN*p?qpuxK=;Rd^i2kmYN!&l| zch0%J&;NZav#WWYbHs(Se#M3GINP%k-*dk_pz;l^!oaXBaIS;(l2{ocG^dBwZkeg7*~=zNa|SeMC4y;LeTPTsf$`b{4L@}UzW0~DvON{la?w2 za(jtYH@`wqX-`yIgrfyu$VZ$yZ3=CVyrD&_aEg~AbZCQMV-=i8ZBX5!2bZC#U5!9j z3|_)%KmYfuOKP2!s@WgF=q6slt}(kNQZp0T*JS+T?AWPXxTEtrlbawQ_{7Xw9()wZ zJA7bMZC;4sQ+iJg!y{UrdDGbJ3XRhqB9Ytq-DuW{#)6Q+82K#>6w3z^qu6F9dL z$8-~Nz-^76D0zd<-o7fofA-%0h|Ru62^HS~GZwrd*SH=J#>M67uxJg^CKSRJE_(oS z(P7UHh9H>+zUWb|=}~fNxOpFCd(7NkYv0^NtIxGsCR5F9fX$y%%;u1w9bDQ;RvMuM zmQ8Ot!68rCU-EdB?D>p;aypACwn@btW}`&Z1aAh>?D@IThE%4}Oiqi|YFz4}Zoh;x zo`c{sR5mp9BA9<+Mdlewv~oODQ}0+3Tr!KG83YdF(?knIs0d689`CYA_d%-7bk=7y z-g+7^9sC&UFN|+Edp3;(a!!MDNh!HcOcrcEJAN@8ygD>Vn$FhADy>X8MyH5}HmnjI zR0YnaIM=!~d&cYa(2)oSvqaZ&yT)S!B=c(Z`g?{?0WGIkXXD~EvRPP=jGR*~F3$QW z2`TFex+fOxEWbEIjmkTD?fXWTU}A&;Gi=erce!Qf2`X;B#Th|9=BxM0-A|hRM8Im* z-sRl9jQf%Wu!{sqoxDQAYFtt)A3k{WSwJE9aiw<1OC$l{v;h__r%`hsAoN0(%%>yv zaWrPRO&YAo)mj(tbt4G!MLmLi6VG_DWY^G+fJh+!+$%`l5*F8{v88+mqz(xVHH7AG zOJ&yZ2uLzH*;rvQrR+Ymn|q#MPs)&N+nI={4m=+50pPv@7^Ao|lG4!2X2ct5erjNYn5RF3sG`EVr`C`z9-{ zZryJct*t$N(8Zd2(0zCk$=`1dR%v&5_7LP%bi*tganTuQq@1nTx|W1WyO%rN4v3`n z1p<|j-v#K9-f`|X@LUV=7CyJ$Q#7o#PQ2I8Tc;@D%%oj4C9lJ@Nn#r1pM_Rg${10a z;~lzNY5dR;$fl=%g`-)KU6d>0LK8{x;IZFl-mTxqhGu+b3QjmM{wgO($AwKiGS?=_xo)>BRZ@0ShPT+{bTXC4bkw(!1$#4{zk*lPdLEv8%9)!f2Y4n` zv5u&EBt&~d&Flp^M#(yIA2ujv#VP6x>rq*OrMs*V`1az;g@qY37gZ1ygtwyF z&Q%D!3~C5pe9ctMya>*KiqTm82}Fwmfr6V>mez>bh6hCb=taSI4M)t9@OQf&G?R*r zF(M91!q%6vtY(Ok)zKJ|ldYSiTd9gXeug%NEc6E#+&@f^c?&Fqs9&fXZy*bpQ&$d& z#3+)K^#zWv!8u>XqPn3R+nZUOj#(l$s^imYGt($`M(#9ZBzx)@$?&u{;cQZ$O^*XD ziputQWN&40XmixRW>y(J*ue^jXOasPPcRwFb>Ji1-0b!`w!G!Z^DS5o-+*Lt8~)4I z_M^rM{U|VY(pvJ`Ad-L0U8>}9G?UeAXy%JeU!bgb#EMjgeWL*i0hj=*W*AJ+D|Ak3 zZMAGL>zFF61n+X9A&je4sno~?RAZ)J;B1nmktSRyn+*e|nX4q1HeD4o4LOD4V~dNt z-b>cQCs;JqNW!EoA~Z)>0m%lN*i0}($}E{lJ~i70oPH71fV)`0dykM?lb^{tzcm%7 z+2_UNRf21nVku4K^nDa2D3&WkoZLlXcLmG8#QC4ifSzJOy%`O$ZEEq1P?3V6wonXW zDCpR}$AX)r7eJR4{%(!4w=S%+{GB85e%~x1OB_W_I~j{ydB7v|+hjsgS@sTH-eW0A zr}qIleOr1>KVK4nKmRBwG(R93^xM3R;_V(ZZ{%Af%{Z%Dux+WpLV6FF2+UlQI;a3(M$&>r?Px$N@NUUq2?+3N)NI%dKzP@`5OM6fbw zcKW;+ykhbX5rS!gf2&#XhF;zq5J==Jl!KRNi!lVo+G5>TH`*NOVKsxt(FFE78!k{z zidfK^LL2>X0d8zu7xmm}(1E*mV|_o{fl4ZLUN;guk=_m|Yv$9Q9y8 z*Yo^vDbEPbV6*KxJP**si@~_86fQMv*M)1D27Id}d>CY5*AR2#SSL`8r`X-#2nYeg zFsPwtbRg;A)=&dg7PgHKUZD{|uxYHr4oof2Eg2NX%eNgKs;5SjY5GPnrhS;f6RP(_ z=%}=hKeCJaqmy@BbVZ$j%V;wL-$EC`v3R zZcBtUP^Z(n#xA=Yw1YwyTEUxQ8Z`n&638BY#9<0tt|@wdtlKf{*8eBcV2TzV8W7jJo;_qbDUse!(hpoz_)8~+*?Z&y0Y z&{y)-m?;diHC^=gq~nYXf_`V;S*6z~KeGgQIl#8V#S^_##mG+k&@jL|ebTx!W?mn@ zg?ER;E35Wgxly`r)t@o>;bJ1!j^DXAzGrI1N#%%)$DWf_5JYBH3yYH6(i{?NRtEc2;6*`c$aO2p5F#A*EH z5=qJ~)AqlVQjnm%)`n~es)3`0+MPCAB3cwu2O z53=umQ1;bBaOqxVZ<)}eIabIf^SV?F_u+c?NRAUnAQ7vMo}anshE9t!B$S(%cG#7F zQd6fvE+>N54;sVrI%lh;I=IV>;=Z%;jnqq{HkQ{GO$i_}-S!3}&60l|k{qJezj&M> zRYYaRbmRWNrCOXJE<|iOScs0(_*b}V7sNx67_O`B$9*MI*0X)5zfb{lklZ4M+wQ0j z1#Cixq&?phnpX&C%D+YRIb7sR)>UEukfpt^{B7}khiu%fgdeq_W8rT9a&T+|Y| z+6w>7mCCxE_kF7^sQNaQ$z105)$n22P&T-Bx4qp+L6IjU<3>ol^FFjowvgtsv*jx-$-ACr|^s5E%MohDfg9!EM-p~Kz zzyIW$d-wSFPp?OP8qKaIr#wl_2PN+6A`Cqie_;<7<>S6BM&U%b)IwgqEeDd4M>1?N zVcG6vF&^6>HNPxtW<-l9A%nWD@fi2Sab&oUSwgXR@hco&t59P9$zZy=F(RHr~`SskSUfi3&8LbDD)tmGt zAp+L1L%=%eT!VnM#l8Uo7JsGrT8+iD9tM_D0?eQyhd6_9Yw1W5(>d#+_E2nalqO%=IH1uKiy^1mJn%dv42**D;@9F1)` zTyEoG1r0vG2_RO`Z#_gTN+k|U6m#efFtK`*|20so-eh@f-rZjL|*>4NFPN4oso7WYpX-AlL*3m@Q+G(9_njxZJNV(5^VMAB2TP0ykJW zKdjIKVhUiqdI(1Wz;i8*7l&+mO4%cS?_rO#1k6iqZ7-Mq#IvJm}sQ<;%a)!EQO8~%nX#}tv7{3qMj38{d z1|E!qHKv>@g_1YK?YDydVyFivdI!i~znaIg(qMm<4@gVd+Cj%FhSWJfq;?X4HV~x0tTmX zD%A^azV{MfLoh}(_sE|m(ad`GkOzH{6w{-Tp1F#Q&!}GPztS%^M_`trJw!k+dgHp3#(ZZ*mwc{y@Y$;z_YM*Bd_2p6Q&7wyxzEj+I9*?lsZP zv|A9K#4&_*q3tn*rpXWE?h($EL3h~;B9dqoNKChZI8$mokG2q|X_*^z1q%F3@1?cG z>{z-b%vnqFE|l`VzbW=F*F-u~+P3idj-bwzXv+jdhyN5<679?{ksITk>3g+!J7I!a z?^i>z2IQF%PGX+vBd(^#En}#QassQg4(ORDls*}Sxzo!Wl$`vy0{6@ck$in9@|kC> zHKPGXE5OfI5b$R88vL_xK*15J{n&iO>&pDd>`rm=c`a-o&sbnkd2honb$p1zjIDj; zTs`$|pwL1eZ&-{rH`YIJ4`P3gHYqFiFCo9$Kj7bsyeG(LvIWAB>;~rsvDgO&@%H*+ z3pb6k6lP-X0vWA0iE01`0J&dzdb$i4jlA7JL_gGU6{(9Pq+6)g`m-x1kS@2wbyG?b zoZ2G58vwhtd9o1#N*JPucs074(uttUBqB<~yDe`WISUJTb{NzPuj(4rs`<`gB(%J{ zF{-Yc-<7E~m!usJUq1}uEx`cOUF&1KagmTCIZuGM>ou$KVEv?l2b@8xJ=<*)C-@Et z5$EXke8PO&3kJ9f>P;z|A_P|Jsc&jvy=f~Io7u2Xmyk7H0`}%RB1e!Ln+xt@jd1kYei|SetaT zC|vnw$gKnD2+L6v^vyRjj5_p=DoM)cI#k14EsNc1r1*oQKVVC{7;{SV{YbU~y4@ZD{f0v@tIOBpT5WD`=cH6HQ38bI5UQ zCeU$SfAn=`vMqX?fBe3oROSkXz7LgQ(c2J2O3sB(9SE1FrLRH}k08^>jeO>Y!VT|g z;|=&kONRBek9xUDy_2Hl5FhcsmS$XQMk6&j5I9%sWYmPl&~odmh|+3E2q|`4#^FF( z$`Jw0C0blFRea1p5LN1sfbuupq-|E;HUhzXD>>WW}B|0J?MXjG^xM@gBu&!0zPtm2z!6FytD& z^h6oj{>lPCSll~5iD?JuZq#_n*)`yIt0&0EZOPf;E+5D3NP2!HSwG;R^x|j1 zrp^v%xd!@83jh_Aj;c|3a>hM@O|AwTyPN16-}6Mz)dGRX%7)NE*l}AO%I&xpVC?A{ zLxL;-%11Ka4pWshwFIMgwvP4u{D@g=&Y3+^wIXN&?nM?U0lF88bv@ucD_QNCK;P2` z9B&A}mnf8KG}NIa0rlV>6oIeN#eHcJQbc+jXfgX71fSise&G>2w2YA1Q(q)w6WK`| zKB>-hmyditYDor`hcYXW`20(sg_nlp8_KZ9SRacIY~3J@{ehYO)kuAm^2-3b?$$7T zINNc^b9S-Z1ci@`+(0Nltla)u_<_0v>?-6k3sw6=Km^;AZ3!7K&p(f^H#MLHA$QhB zD+$}};Cehk>BT|V$!;ME`X%2>VG7bdHnuB0ccr-E(_Y>J7L1|_z8#wfq8n;+4~L>j z(?n~YI~nHV~3tL1iQk?-F zTnSp3=qxJ=CS`rx!UT~up$i*5Tl26Czc6X!QRw>92cN9QF>JiBM_4|OY?p#EO!VN{ zSnCuk(XI_>m}u0*G)y#E1#8%7-jbjZ3T{-F#4O-Vy(8-~(AcjOib5044RFF02#1ZA zi!^Tma+n}v%=-E`hYcFbX4J!a-Bqa1 zHwZ~TFiuCzWl;-_##wMg@)?xm+y?!SByKR2A6|6rF`Jg)HtO08qw6DrRPb+Ow?@C+ z{7y#tVnhpO0=jKt)@{+kCT0p(sbmMJSM<@S7nxiWDg-c&-ub%V*qqKLBN-c@^5JJe z-huXrNhM;tX_RwgK*uOZvGkg!ElpI%sFRIsv~!r_S$9Z4ck}|WJoquKh42`$WNAd> zKu{Od6F5-6sAD`98!ek8Q)0ssPRyf~gGj}>s2)i`K1K{BN8&d+7WXmg6A!{h%lPR20f7^9+~+J5Mzmct>7Ypi*y~Vi3N~1 zzvRKlODjc}BSt2=A6h)_WuTFXKG)+$COWMN9VtEOfv^FkB93aGyy6q_BZ|*$x>(8P`m^aFe(k{IX5Jt!@S#>qB5h zk;mFFnEHCN=$I)Fa&vKooI@(pI(`;KA)jteMibShwiZu~sg?)PgllQ2sVCEqUGZgt zj38oq(-_>lXpj2 zm80`ByrN=2GPw>ZaYnXsi4ovAEtpI%PN!=}4WLY8yhw_F?f@&(zf`N&T$Qj@KAg@` zsT6q?Cvo`Q*0~XI5a)ZYqz%7&a&8RQfMz;N&H)9Tmg8o&gkbS2-3g}W0?tsrC@ef} zG9aM7pp;|sB}+Ogb3*74G^tv0wH_i!kJ56HM$t?#ZGkjBx00bw&44*Jl~P=tgTx7E zgKaXyoySPWUcF4;8)bj~Nsn8%7zjXwdks1KyF(SVX7@(Wnx52$V{3X^ z9M1Ju$RIAUU#8T^>pSMaLTt8aHZV5h2BC&qd3@bK*=*>Fc4@?INHx1qfq84j4>!Zw zbPF%IDpm^ii(s7H1aQ-z#U|aXh1`@k>S%f&g1QMh|Chm;#pI+BkKwL@5b~xdoJe*V zSj|-+oP6b-VQ`8f-VB11@4qtyPI*6d^}_);$=99XZwg;7CbR{?xGb!xs`cSF$<^JU zZwgB*z&H6O!g0W=e=zo@0)l-w^d^v!*kylb>#9$13XOK$|C> zO&CBlToYx}wO7JNW~~t*!aWKyhd8>jXg+nZGLMlX8qN#2fG= z-Jr(h04kMu)_x)M0lpCnH$6V~$dhA$r7r+Oh+g?jp899Dmwx`S+B^eKncI%cmruiY zGF^N*n$Pi->n!FA9s%jok_7fAIdSj8BcL=Bc^K_KHtc&O%rksz7{GGm>qhjeYPz#4 z*Ezo&0YKAyX)V7c^Zwi227UM9*5{|lTj13-v3P*HC9wJ}NzAU!<+~B_J=kxxg&9B~ zNzZ@&ad!`2Li?E!7)B!_$tS0XzX2SIAa-#$J5=YRdT^M84M?;ijDSU%PN{nKCbUwjcm2)o?J3jj)QVVWVNCb=d?tEKo) gfBm0$!8POrvGw3rfAq)f_51kmAMx+=f8cNb17h3yE&u=k diff --git a/docs/source/tutorials/sg_execution_times.rst b/docs/source/tutorials/sg_execution_times.rst deleted file mode 100644 index 0bb42ae96..000000000 --- a/docs/source/tutorials/sg_execution_times.rst +++ /dev/null @@ -1,40 +0,0 @@ - -:orphan: - -.. _sphx_glr_tutorials_sg_execution_times: - - -Computation times -================= -**00:01.107** total execution time for 2 files **from tutorials**: - -.. container:: - - .. raw:: html - - - - - - - - .. list-table:: - :header-rows: 1 - :class: table table-striped sg-datatable - - * - Example - - Time - - Mem (MB) - * - :ref:`sphx_glr_tutorials_template_tutorial.py` (``template_tutorial.py``) - - 00:01.101 - - 0.0 - * - :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` (``rl_fundamentals_forge_tutorial.py``) - - 00:00.006 - - 0.0 diff --git a/docs/source/tutorials/template_tutorial.codeobj.json b/docs/source/tutorials/template_tutorial.codeobj.json deleted file mode 100644 index 21bdbcd5d..000000000 --- a/docs/source/tutorials/template_tutorial.codeobj.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "torch.rand": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "builtin_function_or_method" - }, - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "rand" - } - ], - "x": [ - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "Tensor" - } - ] -} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.ipynb b/docs/source/tutorials/template_tutorial.ipynb deleted file mode 100644 index ae808fece..000000000 --- a/docs/source/tutorials/template_tutorial.ipynb +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n# Template Tutorial\n\n**Author:** [FirstName LastName](https://github.com/username)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Item 1\n * Item 2\n * Item 3\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0\n * GPU ???\n * Other items 3\n\n\nTo test your tutorial locally, you can do one of the following:\n\n* You can control specific files that generate the results by using\n ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable\n respects regular expressions.\n For example to run only ``neural_style_transfer_tutorial.py``,\n use the following command:\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" make html\n```\n or\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" sphinx-build . _build\n```\n* Make a copy of this repository and add only your\n tutorial to the `beginner_source` directory removing all other tutorials.\n Then run ``make html``.\n\nVerify that all outputs were generated correctly in the created HTML.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Overview\n\nDescribe Why is this topic important? Add Links to relevant research papers.\n\nThis tutorial walks you through the process of....\n\n## Steps\n\nExample code (the output below is generated automatically):\n\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import torch\n\nx = torch.rand(5, 3)\nprint(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## (Optional) Additional Exercises\n\nAdd additional practice exercises for users to test their knowledge.\nExample: [NLP from Scratch](https://pytorch.org/tutorials/intermediate/char_rnn_generation_tutorial.html#exercises)_.\n\n\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conclusion\n\nSummarize the steps and concepts covered. Highlight key takeaways.\n\n## Further Reading\n\n* Link1\n* Link2\n\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.18" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.py b/docs/source/tutorials/template_tutorial.py deleted file mode 100644 index 4018aa1b1..000000000 --- a/docs/source/tutorials/template_tutorial.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -""" -Template Tutorial -================= - -**Author:** `FirstName LastName `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Item 1 - * Item 2 - * Item 3 - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0 - * GPU ??? - * Other items 3 - - -To test your tutorial locally, you can do one of the following: - -* You can control specific files that generate the results by using - ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable - respects regular expressions. - For example to run only ``neural_style_transfer_tutorial.py``, - use the following command: - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html - - or - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build - -* Make a copy of this repository and add only your - tutorial to the `beginner_source` directory removing all other tutorials. - Then run ``make html``. - -Verify that all outputs were generated correctly in the created HTML. -""" - -######################################################################### -# Overview -# -------- -# -# Describe Why is this topic important? Add Links to relevant research papers. -# -# This tutorial walks you through the process of.... -# -# Steps -# ----- -# -# Example code (the output below is generated automatically): -# -import torch - -x = torch.rand(5, 3) -print(x) - -###################################################################### -# (Optional) Additional Exercises -# ------------------------------- -# -# Add additional practice exercises for users to test their knowledge. -# Example: `NLP from Scratch `__. -# - -###################################################################### -# Conclusion -# ---------- -# -# Summarize the steps and concepts covered. Highlight key takeaways. -# -# Further Reading -# --------------- -# -# * Link1 -# * Link2 diff --git a/docs/source/tutorials/template_tutorial.py.md5 b/docs/source/tutorials/template_tutorial.py.md5 deleted file mode 100644 index 60b156970..000000000 --- a/docs/source/tutorials/template_tutorial.py.md5 +++ /dev/null @@ -1 +0,0 @@ -1d0dbb374fded9aaf6cf60085291fe43 \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.rst b/docs/source/tutorials/template_tutorial.rst deleted file mode 100644 index e4768c072..000000000 --- a/docs/source/tutorials/template_tutorial.rst +++ /dev/null @@ -1,152 +0,0 @@ - -.. DO NOT EDIT. -.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. -.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: -.. "tutorials/template_tutorial.py" -.. LINE NUMBERS ARE GIVEN BELOW. - -.. only:: html - - .. note:: - :class: sphx-glr-download-link-note - - :ref:`Go to the end ` - to download the full example code. - -.. rst-class:: sphx-glr-example-title - -.. _sphx_glr_tutorials_template_tutorial.py: - - -Template Tutorial -================= - -**Author:** `FirstName LastName `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Item 1 - * Item 2 - * Item 3 - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0 - * GPU ??? - * Other items 3 - - -To test your tutorial locally, you can do one of the following: - -* You can control specific files that generate the results by using - ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable - respects regular expressions. - For example to run only ``neural_style_transfer_tutorial.py``, - use the following command: - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html - - or - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build - -* Make a copy of this repository and add only your - tutorial to the `beginner_source` directory removing all other tutorials. - Then run ``make html``. - -Verify that all outputs were generated correctly in the created HTML. - -.. GENERATED FROM PYTHON SOURCE LINES 56-68 - -Overview --------- - -Describe Why is this topic important? Add Links to relevant research papers. - -This tutorial walks you through the process of.... - -Steps ------ - -Example code (the output below is generated automatically): - - -.. GENERATED FROM PYTHON SOURCE LINES 68-73 - -.. code-block:: Python - - import torch - - x = torch.rand(5, 3) - print(x) - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - tensor([[0.0394, 0.0994, 0.5448], - [0.9781, 0.6839, 0.4422], - [0.9133, 0.0060, 0.0412], - [0.7065, 0.5727, 0.8577], - [0.4415, 0.9282, 0.3205]]) - - - - -.. GENERATED FROM PYTHON SOURCE LINES 74-80 - -(Optional) Additional Exercises -------------------------------- - -Add additional practice exercises for users to test their knowledge. -Example: `NLP from Scratch `__. - - -.. GENERATED FROM PYTHON SOURCE LINES 82-92 - -Conclusion ----------- - -Summarize the steps and concepts covered. Highlight key takeaways. - -Further Reading ---------------- - -* Link1 -* Link2 - - -.. rst-class:: sphx-glr-timing - - **Total running time of the script:** (0 minutes 1.101 seconds) - - -.. _sphx_glr_download_tutorials_template_tutorial.py: - -.. only:: html - - .. container:: sphx-glr-footer sphx-glr-footer-example - - .. container:: sphx-glr-download sphx-glr-download-jupyter - - :download:`Download Jupyter notebook: template_tutorial.ipynb ` - - .. container:: sphx-glr-download sphx-glr-download-python - - :download:`Download Python source code: template_tutorial.py ` - - .. container:: sphx-glr-download sphx-glr-download-zip - - :download:`Download zipped: template_tutorial.zip ` diff --git a/docs/source/tutorials/template_tutorial.zip b/docs/source/tutorials/template_tutorial.zip deleted file mode 100644 index 321b881c49affa7b268465506f2f059b02457417..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5757 zcmd5=-EJGl6;_kBKo`AC(c6O+3P^%TSyBtMOdLmb9?n!cPc;CvYn)n#Vjeq)8KRR^zreF(_?~A@Ca0L)_Tg7lNr>Xk{w~~LYj9{cB3ec z_igtF6YCo~Ax3fd(6Xh!ZLKKotMDy^;#QU=9?d^?~pVY%{W{bTV zrW&mJG)Rso zo~E{@O9g*wDVzz6z^j*OQB^Z8#JoDClTQArRu$M+Gl5vQLr)=%_S=2U?}L@!hwq}H zR+aD6#yUhvwmSB&iDfe}-d(%?^vt?!LKg?!2i*q?t-k30o*q4Vv>?1h+#Rv&DuTAr znI+g6+=G;9;z-(NLhI>n@LDDe<(90Gvs{o!BCYMEGUFcNOY{|7%ZuQD@2sY(l$jc- zEM!%M?1B5o(nyE24rz&8uC=ddIHkG*8&)q(zc@NMIezt3(my&oJAU;dO-Y&y<*X^B z@pK_jN<%HX$bCsGYv;4f3EZIo*gH9{wQwY_5Y(zt)}W}c;E83LK$SurORhFBPEVnN zk+l;#2_=(wCrmYxZn=)rbeB0GL4zT4N5J-?K>flYm_~+jQFey#<2kaknzV_uqW8o2 zPKhxU;#|_i7dkMp?tdVnDksWZb%wRlIdv%s4;odqe10c|LO= zD$K$9c@n|3+=J4g94mw9C$oN)F7*~C3wwbCAY>cakecoG5GsV)g!rWC+zV-n_VWkn z)Mz?Ot+2QDWsQ(uqNL`zmP2pHZ49w(dRyisk}#()&z_%jLlcWO*EP^iUtXX`D|v~> zPBTKg{#;g>Q$vYX1XZ{s_`{Z{5ml7jv5bF2NATo{GUq&nG)V^ z`+w^yx)d6lxWoC0v-Nlq9A7${A?;DLU5v2p)XTDJl?#;PrVhd(vc>VT!IduNh%lcS;i*2~hP zn1$YEm#Nt^9bZY8sR~Vd;ruoTE8oF#_O@7TFP-2cM4~e^e9-0`SGo)F2wf8XQI5`y zz0@)v;~d zmsun|a0z`0bfW{pIhQC+bPjPTrcJ^=sa+`VS5oBM@o&T^2-^wyzmM73(?QgqL#=-U zYW?#cr~b2_Zfx-X&u@iVs+^kPU(kl)Oln=lJ$lU@nV7^QkP|;GC8lD|`Z#oSo`Q$7 znzJd%fRa2Gm|E-kbOFqk3ok%#xiLSSg`x!P0(m5KB^yCs&-cv=DyZ^iPMq~Z?3t+- zRQm;rtJKzOHAKJC%E-83_9lEcEL-8%O+k>(b9-qP<}B;G(B?M50|P|43g!l8V3xt# z>zm;1&DLAM+cq;;0&o12&0Gh3n{fiOI}lPU6DwN;H{aSqxK+YoyXozU-i_ksjNeDe z>p*VKVuudQ?E$D4vzNhZE zxxjU-{5r85^9!%?bI-pEl;d4cDu6kDz6XZm?eFEt|F08s8Go`Nw;|AhX(8zMmoIl9 zR91Bz@N*t|z=&1!xj38Gi$1rL?t3TAeXt1DW)JtIFw^FM_H> z^P3Q_Xbf2Ly~`1TwA#9i8r``%>^x4pRVJ|5qGbB Type[T]: """ @@ -89,7 +119,7 @@ def options( actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) await actor.shutdown() - + * Default usage without calling options .. code-block:: python @@ -100,14 +130,13 @@ def options( attrs = { "procs": procs, - "hosts": hosts, "with_gpus": with_gpus, "num_replicas": num_replicas, + "mesh_name": mesh_name, "_extra_config": kwargs, } - return type(cls.__name__, (cls,), attrs) @classmethod @@ -130,11 +159,12 @@ async def as_service( "hosts": cls.hosts, "with_gpus": cls.with_gpus, "num_replicas": cls.num_replicas, + "mesh_name": cls.mesh_name, **cls._extra_config, # all extra fields } cfg = ServiceConfig(**cfg_kwargs) - logger.info("Spawning Service Actor for %s", cls.__name__) + logger.info("Spawning Service for %s", cls.__name__) service = Service(cfg, cls, actor_args, actor_kwargs) await service.__initialize__() return ServiceInterface(service, cls) @@ -154,28 +184,6 @@ async def setup(self): """ pass - @endpoint - async def set_env(self, addr: str, port: str): - """A temporary workaround to set master addr/port. - - TODO - issues/144. This should be done in proc_mesh creation. - The ideal path: - - Create a host mesh - - Grab a host from host mesh, from proc 0 spawn an actor that - gets addr/port - - Spawn procs on the HostMesh with addr/port, setting the - addr/port in bootstrap. - - We can't currently do this because HostMesh only supports single - proc_mesh creation at the moment. This will be possible once - we have "proper HostMesh support". - - """ - import os - - os.environ["MASTER_ADDR"] = addr - os.environ["MASTER_PORT"] = port - @classmethod async def launch(cls, *args, **kwargs) -> "ForgeActor": """Provisions and deploys a new actor. @@ -195,17 +203,14 @@ async def launch(cls, *args, **kwargs) -> "ForgeActor": procs=cls.procs, hosts=cls.hosts, with_gpus=cls.with_gpus, + mesh_name=cls.mesh_name, ) proc_mesh = await get_proc_mesh(process_config=cfg) actor_name = kwargs.pop("name", cls.__name__) - actor = await proc_mesh.spawn(actor_name, cls, *args, **kwargs) + actor = proc_mesh.spawn(actor_name, cls, *args, **kwargs) actor._proc_mesh = proc_mesh - - if hasattr(proc_mesh, "_hostname") and hasattr(proc_mesh, "_port"): - host, port = proc_mesh._hostname, proc_mesh._port - await actor.set_env.call(addr=host, port=port) await actor.setup.call() return actor diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index 402b9a419..a1fe327e1 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -486,6 +486,10 @@ async def _get_replica(self, sess_id: str | None) -> "Replica": ) async def stop(self): + """ + Stops the service and all managed replicas. + This method should be called when the service is no longer needed. + """ logger.debug("Stopping service...") # Signal shutdown to health loop self._shutdown_requested = True From 76434e33838cf04b58a341174f50bf96eedee617 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 12:30:08 -0700 Subject: [PATCH 05/33] Update --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 78fd09634..c6636dcfb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,7 +39,7 @@ jobs: run: python -m pip install torch==2.9.0 --index-url https://download.pytorch.org/whl/test/cu130 - name: Install monarch shell: bash -l {0} - run: python -m pip install monarch-no-torch==0.1.0.dev20250826 --find-links assets/ci + run: pip install torchmonarch - name: Install torchforge shell: bash -l {0} env: From d9542a37ce26e015080baa562037b78a6f741f96 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 12:46:35 -0700 Subject: [PATCH 06/33] Update --- .github/workflows/docs.yml | 15 ++++++++++++ docs/source/conf.py | 4 ++-- src/forge/controller/actor.py | 45 ++++++++++++++++------------------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c6636dcfb..2cbec1a42 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -52,6 +52,21 @@ jobs: shell: bash -l {0} working-directory: docs run: | + # Set up library paths to ensure all dependencies are available + # This is critical for monarch and other native dependencies that need libpython3.10.so.1.0 + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH:-}" + + # Also set CUDA paths if needed + if [ -d "/usr/local/cuda-12.9" ]; then + export LD_LIBRARY_PATH="/usr/local/cuda-12.9/compat:${LD_LIBRARY_PATH}" + export CUDA_HOME=/usr/local/cuda-12.9 + fi + + # Verify dependencies can be imported before building docs + echo "Verifying dependencies..." + python -c "import forge; print('✓ forge imported successfully')" + python -c "import monarch; print('✓ monarch imported successfully')" + set +e # Don't exit on error make html SPHINXOPTS="-WT --keep-going" || echo "Build completed with warnings/errors" set -e # Re-enable exit on error for subsequent commands diff --git a/docs/source/conf.py b/docs/source/conf.py index 77ee13383..d34e2c927 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -124,6 +124,7 @@ def get_version_path(): "navbar_center": "navbar-nav", "canonical_url": "https://meta-pytorch.org/forge/", "header_links_before_dropdown": 7, + "show_nav_level": 2, } theme_variables = pytorch_sphinx_theme2.get_theme_variables() @@ -176,7 +177,6 @@ def get_version_path(): "plot_gallery": "True", "promote_jupyter_magic": True, "backreferences_dir": None, - "write_computation_times": True, "show_signature": False, - "write_computation_times": False + "write_computation_times": False, } diff --git a/src/forge/controller/actor.py b/src/forge/controller/actor.py index baa1bd0d9..4a5cbf173 100644 --- a/src/forge/controller/actor.py +++ b/src/forge/controller/actor.py @@ -24,40 +24,37 @@ class ForgeActor(Actor): """ Base class for Forge actors with configurable resource attributes. + + The initialization sets up logging configuration with rank/size information and + initializes the actor's process mesh reference. The rank and size are automatically + determined from the current execution context. + + Args: + *args: Variable length argument list passed to the parent Actor class. + **kwargs: Arbitrary keyword arguments passed to the parent Actor class. """ - + procs: int = 1 """Number of processes to use for this actor. Defaults to 1.""" - + hosts: int | None = None """Number of hosts to distribute the actor across. If None, uses as many hosts as needed to accommodate the requested processes. Defaults to None.""" - + with_gpus: bool = False """Whether to allocate GPU resources for this actor. Defaults to False.""" - + num_replicas: int = 1 """Number of replicas to create when spawning as a service. Only applies when using as_service(). Defaults to 1.""" - + mesh_name: str | None = None """Optional name for the process mesh used by this actor. If None, a default name will be generated. Defaults to None.""" - + _extra_config: dict[str, Any] = {} def __init__(self, *args, **kwargs): - """ - Initialize the ForgeActor instance. - - Sets up logging configuration with rank/size information and initializes - the actor's process mesh reference. The rank and size are automatically - determined from the current execution context. - - Args: - *args: Variable length argument list passed to the parent Actor class. - **kwargs: Arbitrary keyword arguments passed to the parent Actor class. - """ if not hasattr(self, "_rank"): self._rank = current_rank().rank if not hasattr(self, "_size"): @@ -100,28 +97,28 @@ def options( Examples: * Pre-configure a service with multiple replicas: - + .. code-block:: python - + service = await MyForgeActor.options(num_replicas=2, procs=2).as_service(...) await service.shutdown() * Default usage without calling options: .. code-block:: python - + service = await MyForgeActor.as_service(...) await service.shutdown() * Pre-configure a single actor - + .. code-block:: python - + actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) await actor.shutdown() - + * Default usage without calling options - + .. code-block:: python actor = await MyForgeActor.as_actor(...) From f101a804bd818902347fefede2738870dfaf483a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 13:16:53 -0700 Subject: [PATCH 07/33] Update --- .github/workflows/docs.yml | 17 ++++++++++++++--- docs/source/api_actors.md | 2 +- docs/source/api_service.md | 4 ++-- docs/source/conf.py | 1 + 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2cbec1a42..73b655c98 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -67,9 +67,20 @@ jobs: python -c "import forge; print('✓ forge imported successfully')" python -c "import monarch; print('✓ monarch imported successfully')" - set +e # Don't exit on error - make html SPHINXOPTS="-WT --keep-going" || echo "Build completed with warnings/errors" - set -e # Re-enable exit on error for subsequent commands + # Build docs with -WT (warnings as errors) and --keep-going to see all issues + # Capture exit code but continue to see all errors + set +e + make html SPHINXOPTS="-WT --keep-going" + BUILD_EXIT_CODE=$? + set -e + + # Report results + if [ $BUILD_EXIT_CODE -ne 0 ]; then + echo "❌ Documentation build failed with warnings or errors (exit code: $BUILD_EXIT_CODE)" + exit $BUILD_EXIT_CODE + else + echo "✅ Documentation build completed successfully with no warnings or errors" + fi - name: Upload docs artifact uses: actions/upload-artifact@v4 with: diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index f086591d8..73eae1220 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -16,5 +16,5 @@ as building blocks for complex distributed training systems. :members: :undoc-members: :show-inheritance: - :exclude-members: logger, setup, set_env + :exclude-members: logger, setup, set_env, __init__ ``` diff --git a/docs/source/api_service.md b/docs/source/api_service.md index 878d43327..f6a72d455 100644 --- a/docs/source/api_service.md +++ b/docs/source/api_service.md @@ -8,7 +8,7 @@ .. autoclass:: Service :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop :show-inheritance: - :special-members: __init__ + :exclude-members: __init__ ``` ## ServiceActor @@ -17,5 +17,5 @@ .. autoclass:: ServiceActor :members: call, call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop :show-inheritance: - :special-members: __init__ + :exclude-members: __init__ ``` diff --git a/docs/source/conf.py b/docs/source/conf.py index d34e2c927..1979a42c3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -125,6 +125,7 @@ def get_version_path(): "canonical_url": "https://meta-pytorch.org/forge/", "header_links_before_dropdown": 7, "show_nav_level": 2, + "show_toc_level": 2 } theme_variables = pytorch_sphinx_theme2.get_theme_variables() From 067ded40bf9a386fecf18f548184a5f5f7092767 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 13:18:05 -0700 Subject: [PATCH 08/33] Update --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1979a42c3..af4f0983e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -125,7 +125,7 @@ def get_version_path(): "canonical_url": "https://meta-pytorch.org/forge/", "header_links_before_dropdown": 7, "show_nav_level": 2, - "show_toc_level": 2 + "show_toc_level": 2, } theme_variables = pytorch_sphinx_theme2.get_theme_variables() From 499c9199c6bd019a2180fa6f68cad46dbe92570a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 8 Oct 2025 16:10:15 -0700 Subject: [PATCH 09/33] Update --- docs/source/api.md | 3 ++ docs/source/api_generator.md | 43 ++++++++++++++++ docs/source/api_model.md | 29 +++++++++++ docs/source/api_service.md | 11 +--- docs/source/api_trainer.md | 68 +++++++++++++++++++++++++ docs/source/conf.py | 8 ++- src/forge/actors/reference_model.py | 14 +++-- src/forge/controller/service/service.py | 28 +--------- 8 files changed, 161 insertions(+), 43 deletions(-) create mode 100644 docs/source/api_generator.md create mode 100644 docs/source/api_model.md create mode 100644 docs/source/api_trainer.md diff --git a/docs/source/api.md b/docs/source/api.md index f4ecd01dc..1235f9d4e 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -29,4 +29,7 @@ APIs provide direct access to the underlying distributed components. :maxdepth: 1 api_actors api_service +api_generator +api_model +api_trainer ``` diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md new file mode 100644 index 000000000..8d17ce7fa --- /dev/null +++ b/docs/source/api_generator.md @@ -0,0 +1,43 @@ +# Generator + +```{eval-rst} +.. currentmodule:: forge.actors.policy +``` + +The Generator (Policy) is the core inference engine in TorchForge, built on top of vLLM. +It manages model serving, text generation, and weight updates for reinforcement learning workflows. + +## Policy + +```{eval-rst} +.. autoclass:: Policy + :members: generate, update_weights, get_version, stop + :exclude-members: __init__ +``` + +## Configuration + +### EngineConfig + +```{eval-rst} +.. autoclass:: EngineConfig + :members: + :undoc-members: +``` + +### SamplingConfig + +```{eval-rst} +.. autoclass:: SamplingConfig + :members: + :undoc-members: +``` + +## PolicyWorker + +```{eval-rst} +.. autoclass:: PolicyWorker + :members: execute_model, update, setup_kv_cache + :show-inheritance: + :exclude-members: __init__ +``` diff --git a/docs/source/api_model.md b/docs/source/api_model.md new file mode 100644 index 000000000..94e51478e --- /dev/null +++ b/docs/source/api_model.md @@ -0,0 +1,29 @@ +# Model + +```{eval-rst} +.. currentmodule:: forge.actors.reference_model +``` + +The {class}`forge.actors.reference_model.ReferenceModel` provides a frozen +copy of the policy model used for computing advantages in reinforcement +learning. It performs inference on input sequences and returns logits or +log probabilities for computing KL divergence and other RL metrics. + +## ReferenceModel + +```{eval-rst} +.. autoclass:: forge.actors.reference_model.ReferenceModel + :members: + :undoc-members: + :show-inheritance: +``` + +The ReferenceModel uses a subset of TorchTitan's configuration system: + +- **model**: Model architecture settings (Model dataclass) +- **parallelism**: Parallelism configuration for distributed inference (Parallelism dataclass) +- **checkpoint**: Checkpoint loading settings (Checkpoint dataclass) +- **compile**: Model compilation settings (Compile dataclass) +- **training**: Training configuration for dtype and other settings (Training dataclass) + +For detailed configuration options, refer to the [TorchTitan documentation](https://github.com/pytorch/torchtitan). diff --git a/docs/source/api_service.md b/docs/source/api_service.md index f6a72d455..df2bf3dc8 100644 --- a/docs/source/api_service.md +++ b/docs/source/api_service.md @@ -6,16 +6,7 @@ ```{eval-rst} .. autoclass:: Service - :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop - :show-inheritance: - :exclude-members: __init__ -``` -## ServiceActor - -```{eval-rst} -.. autoclass:: ServiceActor - :members: call, call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop + :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop :show-inheritance: - :exclude-members: __init__ ``` diff --git a/docs/source/api_trainer.md b/docs/source/api_trainer.md new file mode 100644 index 000000000..75aba94f0 --- /dev/null +++ b/docs/source/api_trainer.md @@ -0,0 +1,68 @@ +# Trainer + +```{eval-rst} +.. currentmodule:: forge.actors.trainer +``` + +The Trainer manages model training in TorchForge, built on top of TorchTitan. +It handles forward/backward passes, weight updates, and checkpoint management for reinforcement learning workflows. + +## RLTrainer + +```{eval-rst} +.. autoclass:: RLTrainer + :members: train_step, push_weights, cleanup + :exclude-members: __init__ +``` + +## Configuration + +The RLTrainer uses TorchTitan's configuration system with the following components: + +### Job Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Job + :members: + :undoc-members: +``` + +### Model Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Model + :members: + :undoc-members: +``` + +### Optimizer Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Optimizer + :members: + :undoc-members: +``` + +### Training Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Training + :members: + :undoc-members: +``` + +### Parallelism Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Parallelism + :members: + :undoc-members: +``` + +### Checkpoint Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Checkpoint + :members: + :undoc-members: +``` diff --git a/docs/source/conf.py b/docs/source/conf.py index af4f0983e..da7c0014b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -162,11 +162,15 @@ def get_version_path(): autodoc_default_options = { "members": True, "member-order": "bysource", - "special-members": "__init__", - "undoc-members": True, "exclude-members": "__weakref__", + "private-members": False, } +# Suppress warnings from third-party library docstrings +suppress_warnings = [ + "docutils", # Suppress docstring formatting issues from third-party libraries +] + # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { diff --git a/src/forge/actors/reference_model.py b/src/forge/actors/reference_model.py index cc57e5246..969c686a0 100644 --- a/src/forge/actors/reference_model.py +++ b/src/forge/actors/reference_model.py @@ -13,6 +13,11 @@ from dataclasses import dataclass, field, fields import torch + +from forge.controller import ForgeActor +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer +from forge.util.ops import compute_logprobs from monarch.actor import current_rank, current_size, endpoint from torch.distributed.tensor import DTensor @@ -26,17 +31,16 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig -from forge.controller import ForgeActor -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer -from forge.util.ops import compute_logprobs - logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @dataclass class ReferenceModel(ForgeActor): + """ + Reference model implementation for the TorchForge service. + """ + # Refer to titan JobConfig for enabling more ForgeEngine configuration model: Model = field(default_factory=Model) parallelism: Parallelism = field(default_factory=Parallelism) diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index a1fe327e1..94638c076 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -38,8 +38,6 @@ import uuid from typing import Dict, List -from monarch.actor import Actor, endpoint - from forge.controller.service.interface import _session_context, Session from forge.controller.service.metrics import ServiceMetrics @@ -52,6 +50,8 @@ ) from forge.types import ServiceConfig +from monarch.actor import Actor, endpoint + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -68,13 +68,6 @@ class Service: actor_def: Actor class definition to instantiate on each replica *actor_args: Positional arguments passed to actor constructor **actor_kwargs: Keyword arguments passed to actor constructor - - - Attributes: - _cfg: Service configuration - _replicas: List of managed replica instances - _active_sessions: Currently active sessions - _metrics: Aggregated service and replica metrics """ def __init__( @@ -609,12 +602,6 @@ class ServiceActor(Actor): actor_def: Actor class definition to instantiate on each replica *actor_args: Positional arguments passed to actor constructor **actor_kwargs: Keyword arguments passed to actor constructor - - Attributes: - _cfg: Service configuration - _replicas: List of managed replica instances - _active_sessions: Currently active sessions - _metrics: Aggregated service and replica metrics """ def __init__(self, cfg: ServiceConfig, actor_def, actor_kwargs: dict): @@ -1163,14 +1150,3 @@ def _get_internal_state(self) -> dict: def __repr__(self): return f"Service(actor={self._actor_def.__name__})" - - -# Copy docstrings from Service to ServiceActor methods for Sphinx autodoc -# This ensures ServiceActor methods have complete docstrings while avoiding duplication -ServiceActor.call.__doc__ = Service._call.__doc__ -ServiceActor.call_all.__doc__ = Service.call_all.__doc__ -ServiceActor.start_session.__doc__ = Service.start_session.__doc__ -ServiceActor.get_metrics.__doc__ = Service.get_metrics.__doc__ -ServiceActor.get_metrics_summary.__doc__ = Service.get_metrics_summary.__doc__ -ServiceActor.terminate_session.__doc__ = Service.terminate_session.__doc__ -ServiceActor.stop.__doc__ = Service.stop.__doc__ From ef3524477f378478ca65fd55b6bc41a73985bae3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 8 Oct 2025 16:15:29 -0700 Subject: [PATCH 10/33] precommit --- src/forge/actors/reference_model.py | 10 +++++----- src/forge/controller/service/service.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/forge/actors/reference_model.py b/src/forge/actors/reference_model.py index 969c686a0..bfe9f9494 100644 --- a/src/forge/actors/reference_model.py +++ b/src/forge/actors/reference_model.py @@ -13,11 +13,6 @@ from dataclasses import dataclass, field, fields import torch - -from forge.controller import ForgeActor -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer -from forge.util.ops import compute_logprobs from monarch.actor import current_rank, current_size, endpoint from torch.distributed.tensor import DTensor @@ -31,6 +26,11 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig +from forge.controller import ForgeActor +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer +from forge.util.ops import compute_logprobs + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index 94638c076..1413cbba1 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -38,6 +38,8 @@ import uuid from typing import Dict, List +from monarch.actor import Actor, endpoint + from forge.controller.service.interface import _session_context, Session from forge.controller.service.metrics import ServiceMetrics @@ -50,8 +52,6 @@ ) from forge.types import ServiceConfig -from monarch.actor import Actor, endpoint - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From bbd8d8fa758df9dc9e7c51021c30ee611a7856bf Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 08:56:50 -0700 Subject: [PATCH 11/33] Fixes --- .github/workflows/docs.yml | 2 +- docs/source/conf.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 73b655c98..9b16e58b1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -70,7 +70,7 @@ jobs: # Build docs with -WT (warnings as errors) and --keep-going to see all issues # Capture exit code but continue to see all errors set +e - make html SPHINXOPTS="-WT --keep-going" + make html SPHINXOPTS="--keep-going" BUILD_EXIT_CODE=$? set -e diff --git a/docs/source/conf.py b/docs/source/conf.py index da7c0014b..78d274f9e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -169,8 +169,13 @@ def get_version_path(): # Suppress warnings from third-party library docstrings suppress_warnings = [ "docutils", # Suppress docstring formatting issues from third-party libraries + "app.add_node", # Suppress node warnings + "app.add_directive", # Suppress directive warnings ] +# Treat warnings as non-fatal - continue build even if there are warnings +keep_warnings = True + # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { From cba68df8ebb6c7694d9ffe775231be398f175a77 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 10:24:16 -0700 Subject: [PATCH 12/33] Update --- docs/source/api_generator.md | 3 ++- docs/source/conf.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index 8d17ce7fa..6f5630583 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -11,8 +11,9 @@ It manages model serving, text generation, and weight updates for reinforcement ```{eval-rst} .. autoclass:: Policy - :members: generate, update_weights, get_version, stop + :members: launch, generate, update_weights, get_version, stop :exclude-members: __init__ + :no-inherited-members: ``` ## Configuration diff --git a/docs/source/conf.py b/docs/source/conf.py index 78d274f9e..7d7fa1979 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,6 +58,7 @@ def get_version_path(): "myst_parser", "sphinx.ext.autodoc", "sphinx.ext.autosummary", + "sphinx-autodoc-typehints", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", @@ -166,6 +167,13 @@ def get_version_path(): "private-members": False, } +# Autodoc configuration for cleaner signatures +autodoc_preserve_defaults = True # Preserves default values without expansion +autodoc_typehints = "description" # Move type hints to description instead of signature +autodoc_typehints_description_target = ( + "documented" # Only add types to documented params +) + # Suppress warnings from third-party library docstrings suppress_warnings = [ "docutils", # Suppress docstring formatting issues from third-party libraries From f828bc17f74dd2cb6b962457e939600031b9a95f Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 10:49:33 -0700 Subject: [PATCH 13/33] Update --- docs/requirements.txt | 1 + src/forge/actors/trainer.py | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 8846bc62e..78dddcdf6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,3 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 +sphinx-autodoc-typehints==3.4.0 diff --git a/src/forge/actors/trainer.py b/src/forge/actors/trainer.py index 4ffc63001..b9882c6d2 100644 --- a/src/forge/actors/trainer.py +++ b/src/forge/actors/trainer.py @@ -18,6 +18,17 @@ import torch.distributed.checkpoint as dcp import torchstore as ts +from forge.actors._torchstore_utils import ( + DcpHandle, + get_dcp_whole_state_dict_key, + get_param_key, +) + +from forge.controller import ForgeActor +from forge.data.utils import batch_to_device +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer + from monarch.actor import current_rank, current_size, endpoint from torch import Tensor from torch.distributed.checkpoint._nested_dict import flatten_state_dict @@ -39,17 +50,6 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig -from forge.actors._torchstore_utils import ( - DcpHandle, - get_dcp_whole_state_dict_key, - get_param_key, -) - -from forge.controller import ForgeActor -from forge.data.utils import batch_to_device -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -95,6 +95,10 @@ def cleanup_old_weight_versions( @dataclass class RLTrainer(ForgeActor): + """ + RL Trainer implementation for the TorchForge service. + """ + job: Job = field(default_factory=Job) model: Model = field(default_factory=Model) optimizer: Optimizer = field(default_factory=Optimizer) From 60ec7e781ec5f968a2bd9dd59d52dcfa8dbab4f5 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 10:53:38 -0700 Subject: [PATCH 14/33] Precommit --- src/forge/actors/trainer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/forge/actors/trainer.py b/src/forge/actors/trainer.py index b9882c6d2..486286680 100644 --- a/src/forge/actors/trainer.py +++ b/src/forge/actors/trainer.py @@ -18,17 +18,6 @@ import torch.distributed.checkpoint as dcp import torchstore as ts -from forge.actors._torchstore_utils import ( - DcpHandle, - get_dcp_whole_state_dict_key, - get_param_key, -) - -from forge.controller import ForgeActor -from forge.data.utils import batch_to_device -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer - from monarch.actor import current_rank, current_size, endpoint from torch import Tensor from torch.distributed.checkpoint._nested_dict import flatten_state_dict @@ -50,6 +39,17 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig +from forge.actors._torchstore_utils import ( + DcpHandle, + get_dcp_whole_state_dict_key, + get_param_key, +) + +from forge.controller import ForgeActor +from forge.data.utils import batch_to_device +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From 754f7f7a945a7a10b1887d1f9a6ea8cb804948ea Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 11:27:20 -0700 Subject: [PATCH 15/33] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 78dddcdf6..601236d5e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.4.0 +sphinx-autodoc-typehints==3.2.0 From cf66872bae7b8a6fda310cf92a6c6043a0ae32ce Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 11:37:48 -0700 Subject: [PATCH 16/33] Update --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 7d7fa1979..3037f0660 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,7 +58,7 @@ def get_version_path(): "myst_parser", "sphinx.ext.autodoc", "sphinx.ext.autosummary", - "sphinx-autodoc-typehints", + "sphinx_autodoc_typehints", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", From b7b39c1015586bcbacaab2dafaddc83876dbc03d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 11:51:02 -0700 Subject: [PATCH 17/33] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 601236d5e..f783fde7a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.2.0 +sphinx-autodoc-typehints==3.0.0 From dfb2b01dea6918ffc36edd6a08f77e12f0173ea0 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 12:14:07 -0700 Subject: [PATCH 18/33] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f783fde7a..78dddcdf6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.0.0 +sphinx-autodoc-typehints==3.4.0 From 678c82c4ab0664afbed5cc54a854a363502b8170 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 12:22:33 -0700 Subject: [PATCH 19/33] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 78dddcdf6..5b35c89c2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.4.0 +sphinx-autodoc-typehints==3.0.1 From 8162490538dbc42496b3d64dc3f855dbd30beb66 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 13:08:17 -0700 Subject: [PATCH 20/33] Update --- docs/requirements.txt | 2 +- docs/source/api_generator.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 5b35c89c2..f783fde7a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.0.1 +sphinx-autodoc-typehints==3.0.0 diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index 6f5630583..c4c448ec0 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -4,7 +4,8 @@ .. currentmodule:: forge.actors.policy ``` -The Generator (Policy) is the core inference engine in TorchForge, built on top of vLLM. +The Generator (Policy) is the core inference engine in TorchForge, +built on top of [vLLM](https://docs.vllm.ai/en/latest/). It manages model serving, text generation, and weight updates for reinforcement learning workflows. ## Policy From 5d1c5c9b4ae32ea5847a61efda62f802fcb66fe8 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 13:46:45 -0700 Subject: [PATCH 21/33] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f783fde7a..525ca1e86 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.0.0 +sphinx-autodoc-typehints==1.25.3 From 197d24d233be2531204dbfe4ef760072b766d576 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 14:35:55 -0700 Subject: [PATCH 22/33] Update --- docs/source/api_generator.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index c4c448ec0..a0bb67f3d 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -25,6 +25,7 @@ It manages model serving, text generation, and weight updates for reinforcement .. autoclass:: EngineConfig :members: :undoc-members: + :no-inherited-members: ``` ### SamplingConfig From bb6b58524608da79604295a0a0a1ff3b21a29c16 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 10:18:32 -0700 Subject: [PATCH 23/33] Update --- docs/source/_static/custom.css | 108 +++++++++++++++++++++++++++++++++ docs/source/_static/custom.js | 93 ++++++++++++++++++++++++++++ docs/source/conf.py | 7 +++ 3 files changed, 208 insertions(+) create mode 100644 docs/source/_static/custom.css create mode 100644 docs/source/_static/custom.js diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 000000000..db41acaa7 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,108 @@ +/* Custom CSS for collapsible parameter lists */ + +/* Hide parameters in signatures */ +.sig-param-hidden { + display: none !important; +} + +/* Inline toggle button for signatures */ +.params-toggle-btn-inline { + display: inline; + padding: 0.2rem 0.5rem; + margin: 0 0.25rem; + background-color: var(--pst-color-background); + border: var(--pst-color-border); + border-radius: 3px; + cursor: pointer; + font-size: 0.85em; + font-family: var(--pst-font-family-base); + color: #495057; + transition: all 0.2s ease; + vertical-align: middle; +} + +.params-toggle-btn-inline:hover { + background-color: #e9ecef; + border-color: #adb5bd; +} + +.params-toggle-btn-inline:focus { + outline: none; +} + +.params-toggle-btn-inline:focus-visible { + outline: 2px solid #007bff; + outline-offset: 2px; +} + +.toggle-icon { + display: inline-block; + font-size: 0.8em; + transition: transform 0.2s ease; +} + +/* Wrapper for the button */ +.sig-params-wrapper { + display: inline; +} + +/* Old styles for field-list collapsing (kept for backward compatibility) */ +.collapsible-params { + margin: 1rem 0; +} + +.params-toggle-btn { + display: inline-block; + padding: 0.5rem 1rem; + margin-bottom: 0.5rem; + background-color: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + cursor: pointer; + font-size: 0.9rem; + color: #495057; + transition: all 0.3s ease; +} + +.params-toggle-btn:hover { + background-color: #e9ecef; + border-color: #adb5bd; +} + +.params-toggle-btn:focus { + outline: 2px solid #007bff; + outline-offset: 2px; +} + +.params-content { + max-height: 10000px; + overflow: hidden; + transition: max-height 0.5s ease, opacity 0.3s ease; + opacity: 1; +} + +.params-content.collapsed { + max-height: 0; + opacity: 0; +} + +/* Ensure the collapsed parameters look good */ +.params-content dl.field-list { + margin-top: 0; +} + +.params-content > dt { + margin-top: 0.5rem; +} + +.params-content > dt:first-child { + margin-top: 0; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .params-toggle-btn { + width: 100%; + text-align: left; + } +} diff --git a/docs/source/_static/custom.js b/docs/source/_static/custom.js new file mode 100644 index 000000000..415592d30 --- /dev/null +++ b/docs/source/_static/custom.js @@ -0,0 +1,93 @@ +// Custom JavaScript to make long parameter lists in class signatures collapsible +document.addEventListener('DOMContentLoaded', function() { + console.log('Collapsible parameters script loaded'); + + // Find all class/function signatures + const signatures = document.querySelectorAll('dl.py.class > dt, dl.py.function > dt, dl.py.method > dt'); + + signatures.forEach(function(signature) { + // Find all parameter elements in the signature + const params = signature.querySelectorAll('em.sig-param, .sig-param'); + + console.log(`Found signature with ${params.length} parameters`); + + // Only make it collapsible if there are more than 10 parameters + if (params.length > 10) { + console.log('Creating collapsible structure for signature with', params.length, 'parameters'); + + const visibleCount = 5; + const hiddenCount = params.length - visibleCount; + + // Create a wrapper div for the toggle button + const wrapper = document.createElement('span'); + wrapper.className = 'sig-params-wrapper'; + wrapper.style.display = 'inline'; + + // Create toggle button + const toggleBtn = document.createElement('button'); + toggleBtn.className = 'params-toggle-btn-inline'; + toggleBtn.innerHTML = ` Show More`; + toggleBtn.setAttribute('aria-expanded', 'false'); + toggleBtn.title = `Show ${hiddenCount} more parameters`; + + // Collect all nodes to hide (params and text nodes between them) + const nodesToHide = []; + + // Hide parameters after the first 3 + let insertedButton = false; + params.forEach(function(param, index) { + if (index >= visibleCount) { + // Add 'hidden' class to hide the parameter + param.classList.add('sig-param-hidden'); + nodesToHide.push(param); + + // Also hide the text node (comma/space) that follows this parameter + let nextNode = param.nextSibling; + while (nextNode && nextNode.nodeType === Node.TEXT_NODE) { + const textSpan = document.createElement('span'); + textSpan.className = 'sig-param-hidden'; + textSpan.textContent = nextNode.textContent; + nextNode.parentNode.replaceChild(textSpan, nextNode); + nodesToHide.push(textSpan); + break; + } + + // Insert the toggle button before the first hidden parameter + if (!insertedButton) { + param.parentNode.insertBefore(wrapper, param); + wrapper.appendChild(toggleBtn); + insertedButton = true; + } + } + }); + + // Add click handler to toggle + toggleBtn.addEventListener('click', function(e) { + e.preventDefault(); + e.stopPropagation(); + + const isExpanded = toggleBtn.getAttribute('aria-expanded') === 'true'; + + if (isExpanded) { + // Collapse: hide parameters again + nodesToHide.forEach(function(node) { + node.classList.add('sig-param-hidden'); + }); + toggleBtn.setAttribute('aria-expanded', 'false'); + toggleBtn.innerHTML = ` Show More`; + toggleBtn.title = `Show ${hiddenCount} more parameters`; + } else { + // Expand: show all parameters + nodesToHide.forEach(function(node) { + node.classList.remove('sig-param-hidden'); + }); + toggleBtn.setAttribute('aria-expanded', 'true'); + toggleBtn.innerHTML = ` Hide`; + toggleBtn.title = `Hide ${hiddenCount} parameters`; + } + }); + + console.log('Collapsible structure created successfully'); + } + }); +}); diff --git a/docs/source/conf.py b/docs/source/conf.py index 3037f0660..c5cdb8620 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -75,12 +75,19 @@ def get_version_path(): ] sitemap_url_scheme = "{link}" +# Ensure static files use relative paths +html_static_path = ["_static"] + templates_path = [ "_templates", os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), ] exclude_patterns = ["tutorials/index.rst", "tutorials/template_tutorial.rst"] +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_js_files = ["custom.js"] + sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../src")) html_theme = "pytorch_sphinx_theme2" From 8c186b110e765a712aae097ed0dac461b0791456 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 11:14:08 -0700 Subject: [PATCH 24/33] Update --- docs/source/_static/custom.css | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index db41acaa7..89854cc8b 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -11,30 +11,25 @@ padding: 0.2rem 0.5rem; margin: 0 0.25rem; background-color: var(--pst-color-background); - border: var(--pst-color-border); + border: 1px solid var(--pst-color-border); border-radius: 3px; cursor: pointer; font-size: 0.85em; font-family: var(--pst-font-family-base); - color: #495057; + color: var(--pst-color-primary); transition: all 0.2s ease; vertical-align: middle; } .params-toggle-btn-inline:hover { - background-color: #e9ecef; - border-color: #adb5bd; + background-color: var(--pst-color-background); + border-color: var(--pst-color-border); } .params-toggle-btn-inline:focus { outline: none; } -.params-toggle-btn-inline:focus-visible { - outline: 2px solid #007bff; - outline-offset: 2px; -} - .toggle-icon { display: inline-block; font-size: 0.8em; @@ -55,23 +50,18 @@ display: inline-block; padding: 0.5rem 1rem; margin-bottom: 0.5rem; - background-color: #f8f9fa; - border: 1px solid #dee2e6; + background-color: var(--pst-color-background); + border: 1px solid var(--pst-color-border); border-radius: 4px; cursor: pointer; font-size: 0.9rem; - color: #495057; + color: var(--pst-color-primary); transition: all 0.3s ease; } .params-toggle-btn:hover { - background-color: #e9ecef; - border-color: #adb5bd; -} - -.params-toggle-btn:focus { - outline: 2px solid #007bff; - outline-offset: 2px; + background-color: var(--pst-color-background); + border-color: var(--pst-color-border); } .params-content { From 25221e71c2c5bd9fbec807a4f051ae28748e2f2c Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 14:22:52 -0700 Subject: [PATCH 25/33] Update --- docs/source/conf.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index c5cdb8620..55d57680e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -191,6 +191,13 @@ def get_version_path(): # Treat warnings as non-fatal - continue build even if there are warnings keep_warnings = True +# Napoleon settings for Google-style docstrings (from torchtitan and other dependencies) +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_preprocess_types = False + # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { From f3c79f68644aff00f820cb039d6a88cfc99a134d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 15:58:34 -0700 Subject: [PATCH 26/33] Update --- docs/source/conf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 55d57680e..ee9d62148 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -186,17 +186,25 @@ def get_version_path(): "docutils", # Suppress docstring formatting issues from third-party libraries "app.add_node", # Suppress node warnings "app.add_directive", # Suppress directive warnings + "ref.class", # Suppress missing reference warnings + "ref.func", # Suppress missing function reference warnings + "ref.meth", # Suppress missing method reference warnings ] # Treat warnings as non-fatal - continue build even if there are warnings keep_warnings = True +# Don't fail the build on warnings - important for handling third-party library docstrings +# This is especially important when dependencies (like torchtitan) have RST formatting +# that may not be perfect but works with Napoleon extension +nitpicky = False # Don't be overly strict about references + # Napoleon settings for Google-style docstrings (from torchtitan and other dependencies) napoleon_google_docstring = True napoleon_numpy_docstring = True napoleon_use_param = True napoleon_use_rtype = True -napoleon_preprocess_types = False +napoleon_use_ivar = True # -- Sphinx Gallery configuration ------------------------------------------- From bee92c63857a08ab4d6a899df17e51c349e9241b Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 13 Oct 2025 11:45:48 -0700 Subject: [PATCH 27/33] Update docs/source/api.md Co-authored-by: Allen Wang <9057208+allenwang28@users.noreply.github.com> --- docs/source/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/api.md b/docs/source/api.md index 1235f9d4e..5c846de91 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -14,7 +14,7 @@ Key Features of TorchForge include: - **Actor-Based Architecture**: TorchForge uses an actor-based system for distributed training, providing excellent scalability and fault tolerance. - **PyTorch Native**: Built natively on PyTorch, ensuring seamless integration with existing PyTorch workflows. -- **Post-Training Focus**: Specifically designed for post-training techniques like RLHF, SFT, and other alignment methods. +- **Post-Training Focus**: Specifically designed for post-training techniques like RLVR, SFT, and other alignment methods. - **Distributed by Design**: Supports multi-GPU and multi-node training out of the box. From ae05ebfbc15e39a4682f336cc0aa201a4b4211f6 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 13 Oct 2025 12:52:39 -0700 Subject: [PATCH 28/33] Update --- docs/source/api_generator.md | 8 -------- docs/source/index.md | 2 -- 2 files changed, 10 deletions(-) diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index a0bb67f3d..0d9a58ff4 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -28,14 +28,6 @@ It manages model serving, text generation, and weight updates for reinforcement :no-inherited-members: ``` -### SamplingConfig - -```{eval-rst} -.. autoclass:: SamplingConfig - :members: - :undoc-members: -``` - ## PolicyWorker ```{eval-rst} diff --git a/docs/source/index.md b/docs/source/index.md index c450b4ca7..de594e342 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -23,10 +23,8 @@ Key Features getting_started concepts -usage tutorials api -faq ``` ## Indices and tables From 00604944dc9ab4a05ad6b8d4478452c0f244b544 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 13 Oct 2025 13:02:07 -0700 Subject: [PATCH 29/33] Update --- .github/workflows/docs.yml | 5 ++++- docs/source/faq.md | 3 --- docs/source/usage.md | 4 ---- src/forge/actors/reference_model.py | 15 ++++++++++++++- src/forge/actors/trainer.py | 20 ++++++++++++++++++-- 5 files changed, 36 insertions(+), 11 deletions(-) delete mode 100644 docs/source/faq.md delete mode 100644 docs/source/usage.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9b16e58b1..3b2f87b55 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -36,10 +36,13 @@ jobs: run: python -m pip install --upgrade pip - name: Install pytorch shell: bash -l {0} - run: python -m pip install torch==2.9.0 --index-url https://download.pytorch.org/whl/test/cu130 + run: pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu130 --force-reinstall - name: Install monarch shell: bash -l {0} run: pip install torchmonarch + - name: Install torchtitan + shell: bash -l {0} + run: pip install --pre torchtitan --index-url https://download.pytorch.org/whl/nightly/cu130 - name: Install torchforge shell: bash -l {0} env: diff --git a/docs/source/faq.md b/docs/source/faq.md deleted file mode 100644 index d3c027866..000000000 --- a/docs/source/faq.md +++ /dev/null @@ -1,3 +0,0 @@ -# FAQ - -This FAQ covers common questions and issues encountered when using TorchForge. diff --git a/docs/source/usage.md b/docs/source/usage.md deleted file mode 100644 index 2a61c577a..000000000 --- a/docs/source/usage.md +++ /dev/null @@ -1,4 +0,0 @@ -# Usage - -This section covers practical usage patterns, -configuration management, and common scenarios for working with TorchForge effectively. diff --git a/src/forge/actors/reference_model.py b/src/forge/actors/reference_model.py index bfe9f9494..8cd8d9553 100644 --- a/src/forge/actors/reference_model.py +++ b/src/forge/actors/reference_model.py @@ -38,7 +38,20 @@ @dataclass class ReferenceModel(ForgeActor): """ - Reference model implementation for the TorchForge service. + A reference model actor for reinforcement learning (RL) training. + + Based on TorchTitan's engine architecture, this actor provides a frozen model that only runs forward passes without gradient computation. It is typically used to maintain algorithmic consistency in policy optimization methods such as GRPO (Group Relative Policy Optimization) or PPO (Proximal Policy Optimization), where it serves as a fixed reference point to compute KL divergence penalties against the training policy. + + The reference model is loaded from a checkpoint and runs in evaluation mode with inference_mode enabled to optimize memory and compute efficiency. + + Attributes: + + model (Model): Model configuration (architecture, vocab size, etc.) + parallelism (Parallelism): Parallelism strategy configuration (TP, PP, CP, DP) + checkpoint (Checkpoint): Checkpoint loading configuration + compile (Compile): Torch compilation settings + comm (Comm): Communication backend configuration + training (Training): Training-related settings (dtype, garbage collection, etc.) """ # Refer to titan JobConfig for enabling more ForgeEngine configuration diff --git a/src/forge/actors/trainer.py b/src/forge/actors/trainer.py index 486286680..521679104 100644 --- a/src/forge/actors/trainer.py +++ b/src/forge/actors/trainer.py @@ -95,8 +95,24 @@ def cleanup_old_weight_versions( @dataclass class RLTrainer(ForgeActor): - """ - RL Trainer implementation for the TorchForge service. + """A reinforcement learning trainer actor for policy optimization training. + + Built on top of TorchTitan's training engine, this actor provides a complete training + loop for reinforcement learning. It performs forward and backward passes with gradient + computation, optimization steps, and checkpoint management. Unlike the ReferenceModel + actor which only runs forward passes, RLTrainer actively updates the policy model + parameters through gradient descent. + + The trainer supports the same distributed training strategies that TorchTitan does, + including but not limited to, tensor parallelism, data parallelism, and FSDP + (Fully Sharded Data Parallel). It is typically used in conjunction with ReferenceModel + for policy optimization algorithms like GRPO (Group Relative Policy Optimization), + where it optimizes the policy against a loss that includes KL divergence penalties + from the reference model. + + The trainer handles: + - Forward and backward propagation with automatic mixed precision (AMP) + - Optimizer steps with learning rate scheduling """ job: Job = field(default_factory=Job) From 00842a5718fdea2b031f172c18052c12c5330548 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 13 Oct 2025 15:29:24 -0700 Subject: [PATCH 30/33] Update --- docs/source/conf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ee9d62148..f4ba3ddd7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -178,9 +178,13 @@ def get_version_path(): autodoc_preserve_defaults = True # Preserves default values without expansion autodoc_typehints = "description" # Move type hints to description instead of signature autodoc_typehints_description_target = ( - "documented" # Only add types to documented params + "documented_params" # Only add types to documented params ) +# Disable docstring inheritance +autodoc_inherit_docstrings = False + + # Suppress warnings from third-party library docstrings suppress_warnings = [ "docutils", # Suppress docstring formatting issues from third-party libraries From 635bbc1ffbd68d26691477a58081640a7baa55c9 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 13 Oct 2025 15:49:20 -0700 Subject: [PATCH 31/33] Update --- docs/source/api_actors.md | 2 +- docs/source/api_generator.md | 13 +------------ docs/source/conf.py | 16 ++++++++++++++-- src/forge/actors/policy.py | 4 ++-- src/forge/actors/reference_model.py | 29 ++++++++++++++++++++--------- src/forge/actors/trainer.py | 24 ++++++++++++------------ 6 files changed, 50 insertions(+), 38 deletions(-) diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index 73eae1220..e53101b27 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -16,5 +16,5 @@ as building blocks for complex distributed training systems. :members: :undoc-members: :show-inheritance: - :exclude-members: logger, setup, set_env, __init__ + :exclude-members: logger, setup, set_env, __init__, as_service ``` diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index 0d9a58ff4..dab0e029e 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -12,22 +12,11 @@ It manages model serving, text generation, and weight updates for reinforcement ```{eval-rst} .. autoclass:: Policy - :members: launch, generate, update_weights, get_version, stop + :members: generate, update_weights, get_version, stop :exclude-members: __init__ :no-inherited-members: ``` -## Configuration - -### EngineConfig - -```{eval-rst} -.. autoclass:: EngineConfig - :members: - :undoc-members: - :no-inherited-members: -``` - ## PolicyWorker ```{eval-rst} diff --git a/docs/source/conf.py b/docs/source/conf.py index f4ba3ddd7..6d92ee830 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -169,9 +169,9 @@ def get_version_path(): autodoc_default_options = { "members": True, - "member-order": "bysource", - "exclude-members": "__weakref__", + "undoc-members": True, "private-members": False, + "inherited-members": False, } # Autodoc configuration for cleaner signatures @@ -183,6 +183,7 @@ def get_version_path(): # Disable docstring inheritance autodoc_inherit_docstrings = False +autodoc_typehints = "none" # Suppress warnings from third-party library docstrings @@ -224,3 +225,14 @@ def get_version_path(): "show_signature": False, "write_computation_times": False, } + + +def clean_docstring_indentation(app, what, name, obj, options, lines): + if name and name.startswith("torchtitan."): + lines[:] = [line.lstrip() for line in lines] + if lines and lines[-1].strip(): + lines.append("") + + +def setup(app): + app.connect("autodoc-process-docstring", clean_docstring_indentation) diff --git a/src/forge/actors/policy.py b/src/forge/actors/policy.py index 3a1b3e86e..1579fea40 100644 --- a/src/forge/actors/policy.py +++ b/src/forge/actors/policy.py @@ -341,8 +341,8 @@ async def generate(self, prompt: str, *, priority: int = 0) -> list[Completion]: def _preprocess_add_request( self, request: EngineCoreRequest ) -> tuple[Request, int]: - """ (forge/issues/332) Will require attention when we bump vllm versions - https://github.com/vllm-project/vllm/blob/0e3bb543f064eb416bca4f6f3013efa3830b12f7/vllm/v1/engine/core.py#L419""" + """(forge/issues/332) Will require attention when we bump vllm versions + https://github.com/vllm-project/vllm/blob/0e3bb543f064eb416bca4f6f3013efa3830b12f7/vllm/v1/engine/core.py#L419""" if request.mm_hashes is not None: raise NotImplementedError("Support for mm_hash is not implemented yet.") req = Request.from_engine_core_request(request) diff --git a/src/forge/actors/reference_model.py b/src/forge/actors/reference_model.py index f0d15d8ee..2f1b122aa 100644 --- a/src/forge/actors/reference_model.py +++ b/src/forge/actors/reference_model.py @@ -40,19 +40,30 @@ class ReferenceModel(ForgeActor): """ A reference model actor for reinforcement learning (RL) training. - - Based on TorchTitan's engine architecture, this actor provides a frozen model that only runs forward passes without gradient computation. It is typically used to maintain algorithmic consistency in policy optimization methods such as GRPO (Group Relative Policy Optimization) or PPO (Proximal Policy Optimization), where it serves as a fixed reference point to compute KL divergence penalties against the training policy. - - The reference model is loaded from a checkpoint and runs in evaluation mode with inference_mode enabled to optimize memory and compute efficiency. - + + Based on TorchTitan's engine architecture, this actor provides a + frozen model that only runs forward passes without gradient + computation. It is typically used to maintain algorithmic + consistency in policy optimization methods such as GRPO + (Group Relative Policy Optimization) or PPO (Proximal Policy + Optimization), where it serves as a fixed reference point to + compute KL divergence penalties against the training policy. + + The reference model is loaded from a checkpoint and runs in + evaluation mode with inference_mode enabled to optimize memory and + compute efficiency. + Attributes: - - model (Model): Model configuration (architecture, vocab size, etc.) - parallelism (Parallelism): Parallelism strategy configuration (TP, PP, CP, DP) + + model (Model): Model configuration (architecture, vocab size, + etc.) + parallelism (Parallelism): Parallelism strategy configuration + (TP, PP, CP, DP) checkpoint (Checkpoint): Checkpoint loading configuration compile (Compile): Torch compilation settings comm (Comm): Communication backend configuration - training (Training): Training-related settings (dtype, garbage collection, etc.) + training (Training): Training-related settings (dtype, garbage + collection, etc.) """ # Refer to titan JobConfig for enabling more ForgeEngine configuration diff --git a/src/forge/actors/trainer.py b/src/forge/actors/trainer.py index dea743e68..dd85b3c82 100644 --- a/src/forge/actors/trainer.py +++ b/src/forge/actors/trainer.py @@ -96,20 +96,20 @@ def cleanup_old_weight_versions( @dataclass class RLTrainer(ForgeActor): """A reinforcement learning trainer actor for policy optimization training. - - Built on top of TorchTitan's training engine, this actor provides a complete training - loop for reinforcement learning. It performs forward and backward passes with gradient - computation, optimization steps, and checkpoint management. Unlike the ReferenceModel - actor which only runs forward passes, RLTrainer actively updates the policy model + + Built on top of TorchTitan's training engine, this actor provides a complete training + loop for reinforcement learning. It performs forward and backward passes with gradient + computation, optimization steps, and checkpoint management. Unlike the ReferenceModel + actor which only runs forward passes, RLTrainer actively updates the policy model parameters through gradient descent. - - The trainer supports the same distributed training strategies that TorchTitan does, - including but not limited to, tensor parallelism, data parallelism, and FSDP - (Fully Sharded Data Parallel). It is typically used in conjunction with ReferenceModel - for policy optimization algorithms like GRPO (Group Relative Policy Optimization), - where it optimizes the policy against a loss that includes KL divergence penalties + + The trainer supports the same distributed training strategies that TorchTitan does, + including but not limited to, tensor parallelism, data parallelism, and FSDP + (Fully Sharded Data Parallel). It is typically used in conjunction with ReferenceModel + for policy optimization algorithms like GRPO (Group Relative Policy Optimization), + where it optimizes the policy against a loss that includes KL divergence penalties from the reference model. - + The trainer handles: - Forward and backward propagation with automatic mixed precision (AMP) - Optimizer steps with learning rate scheduling From a1972f5800ce4f7771728470b85fdf3662c278fd Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 13 Oct 2025 16:43:47 -0700 Subject: [PATCH 32/33] Update --- .github/workflows/docs.yml | 4 ++-- docs/source/api_generator.md | 2 +- docs/source/conf.py | 20 ++++---------------- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5e8840e28..c0848d2a8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -68,10 +68,10 @@ jobs: python -c "import forge; print('✓ forge imported successfully')" python -c "import monarch; print('✓ monarch imported successfully')" - # Build docs with -WT (warnings as errors) and --keep-going to see all issues + # Build docs with -W (warnings as errors) and --keep-going to see all issues # Capture exit code but continue to see all errors set +e - make html SPHINXOPTS="--keep-going" + make html SPHINXOPTS="-W --keep-going" BUILD_EXIT_CODE=$? set -e diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index dab0e029e..c5aee3eec 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -13,7 +13,7 @@ It manages model serving, text generation, and weight updates for reinforcement ```{eval-rst} .. autoclass:: Policy :members: generate, update_weights, get_version, stop - :exclude-members: __init__ + :exclude-members: __init__, launch :no-inherited-members: ``` diff --git a/docs/source/conf.py b/docs/source/conf.py index 6d92ee830..4e3cec1fa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -186,23 +186,11 @@ def get_version_path(): autodoc_typehints = "none" -# Suppress warnings from third-party library docstrings -suppress_warnings = [ - "docutils", # Suppress docstring formatting issues from third-party libraries - "app.add_node", # Suppress node warnings - "app.add_directive", # Suppress directive warnings - "ref.class", # Suppress missing reference warnings - "ref.func", # Suppress missing function reference warnings - "ref.meth", # Suppress missing method reference warnings -] - -# Treat warnings as non-fatal - continue build even if there are warnings -keep_warnings = True +# Removed suppress_warnings to make the build stricter +# All warnings will now be treated as errors when -W is passed to sphinx-build -# Don't fail the build on warnings - important for handling third-party library docstrings -# This is especially important when dependencies (like torchtitan) have RST formatting -# that may not be perfect but works with Napoleon extension -nitpicky = False # Don't be overly strict about references +# Be strict about references to catch broken links and references +nitpicky = False # Napoleon settings for Google-style docstrings (from torchtitan and other dependencies) napoleon_google_docstring = True From efc74c666d5aae94aa6b8bceb17610c84fa51e4e Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 14 Oct 2025 08:43:51 -0700 Subject: [PATCH 33/33] Fix example line --- src/forge/actors/policy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/forge/actors/policy.py b/src/forge/actors/policy.py index 1579fea40..8e8c8de17 100644 --- a/src/forge/actors/policy.py +++ b/src/forge/actors/policy.py @@ -76,6 +76,7 @@ class Policy(PolicyInterface): use_dcp (bool): Whether to use DCP for NFS-based weight sync. Example: + >>> policy = await Policy.options(procs=1, num_replicas=1, with_gpus=True).as_service( ... engine_args=EngineArgs(...), ... sampling_params=SamplingParams(...),