forked from LLM360/website
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathpretraining.html
More file actions
349 lines (328 loc) · 17.9 KB
/
pretraining.html
File metadata and controls
349 lines (328 loc) · 17.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
<!DOCTYPE html>
<html lang="en">
<head>
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-0FFN6N7318"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-0FFN6N7318');
</script>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no" />
<!-- SEO -->
<title>LLM360 Pretraining Suite | Step-by-step LLM Guides</title>
<meta name="description" content="Access the LLM360 Pretraining Suite for detailed guides on reproducing models like Amber, Crystal, and K2. Ideal for tech enthusiasts and AI researchers, these tutorials are designed to empower your next AI project with knowledge from Petuum and MBZUAI experts."/>
<link rel="canonical" href="https://www.llm360.ai/pretraining.html" />
<meta name="keywords" content="LLM360, LLM Pretraining, Open Source AI, AI Tutorials, Large Language Models, Petuum, MBZUAI, Amber LLM, CrystalCoder, K2, AI Research, Pretraining Techniques"/>
<meta name="author" content="Petuum, Mohamed bin Zayed University of Artificial Intelligence"/>
<!-- Open Graph Protocol -->
<meta property="og:title" content="LLM360 Pretraining Suite | Step-by-Step LLM Guides"/>
<meta property="og:description" content="Access the LLM360 Pretraining Suite for detailed guides on reproducing models like Amber, Crystal, and K2. Our comprehensive tutorials are tailored for both new and experienced AI practitioners, fostering deep understanding of LLM techniques."/>
<meta property="og:type" content="website"/>
<meta property="og:url" content="https://www.llm360.ai/pretraining.html"/>
<meta property="og:image" content="https://www.llm360.ai/images/open-graph-image.jpg"/>
<!-- Twitter Card -->
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:site" content="@llm360">
<meta name="twitter:title" content="LLM360 Pretraining Suite | Step-by-Step LLM Guides">
<meta name="timestamp" content="Explore the LLM360 Pretraining Suite for step-by-step tutorials on Amber, Crystal, and K2. Join us to advance your skills in AI pretraining and contribute to open-source AI research."/>
<meta name="twitter:image" content="https://www.llm360.ai/images/open-graph-image.jpg">
<!-- Schema Markup -->
<script type="application/ld+json">
{
"@context": "http://schema.org",
"@type": "WebSite",
"name": "LLM360 Pretraining Suite",
"url": "https://www.llm360.ai/pretraining.html",
"description": "The LLM360 Pretraining Suite offers in-depth tutorials for reproducing Amber, CrystalCoder, and K2 models, providing AI practitioners and researchers with the tools and knowledge to engage in advanced AI pretraining techniques.",
"publisher": {
"@type": "Organization",
"name": "LLM360",
"sameAs": [
"https://x.com/llm360",
"https://github.com/LLM360"
]
},
"image": {
"@type": "ImageObject",
"url": "https://www.llm360.ai/images/open-graph-image.jpg",
"width": 1920,
"height": 1080
}
}
</script>
<!-- /SEO -->
<!-- Favicon -->
<link rel="icon" type="image/x-icon" href="assets/favicon/favicon.ico" />
<link rel="icon" type="image/png" sizes="192x192" href="assets/favicon/android-chrome-192x192.png">
<link rel="icon" type="image/png" sizes="512x512" href="assets/favicon/android-chrome-512x512.png">
<link rel="icon" type="image/png" sizes="32x32" href="assets/favicon/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="assets/favicon/favicon-16x16.png">
<link rel="apple-touch-icon" sizes="180x180" href="assets/favicon/apple-touch-icon.png">
<!-- <link rel="manifest" href="/site.webmanifest"> -->
<meta name="msapplication-TileColor" content="#da532c">
<meta name="theme-color" content="#ffffff">
<!-- Style -->
<link rel="stylesheet" href="assets/css/main.css" />
</head>
<body class="is-preload">
<!-- Sidebar -->
<button class="toggle-btn" id="toggleBtn">☰</button>
<section id="sidebar" class="sidebar">
<div class="inner">
<nav>
<a class="alt" href="index.html">
<figure class="hover-rotate">
<img src="images/logo-highres.png" alt="logo" />
</figure>
</a>
<h2>LLM360</h2>
<ul>
<li><a href="index.html#one">Models</a></li>
<li><a href="evaluation.html">Performance and Evaluation</a></li>
<li><a href="index.html#two" >LLM360 Suites</a></li>
<li><a href="index.html#three">Papers</a></li>
<li><a href="index.html#four">Blogs</a></li>
<li><a href="index.html#six">Get in touch</a></li>
<li><a href="community.html">Open-source Communities</a></li>
<li><a href="about.html">About</a></li>
</ul>
</nav>
</div>
</section>
<!-- Wrapper -->
<div id="wrapper">
<!-- Intro -->
<section id="top" class="wrapper fullscreen fade-up">
<div class="inner">
<h1>LLM360 Pretraining Suite</h1>
<!-- <p>Under construction...</p>-->
<p>LLM360 Pretraining Suite is a series of step-by-step guides to reproduce each of our models, for tech enthusiasts, AI practitioners, and academic or industry researchers, to transfer knowledge on LLM pretraining techniques.</br>
Our tutorials are currently in active development and will be released in the upcoming weeks. Stay tuned!</p>
<form action="https://formspree.io/f/xzbnkzqg" method="POST" name="Tutorial Waitlist">
<div class="fields">
<div class="field half">
<input type="email" name="demo-email" id="demo-email" value="" placeholder="Email" required/>
</div>
<div class="field half">
<ul class="actions">
<li><a href="" class="button submit">Notify Me</a></li>
</ul>
</div>
</div>
</form>
</div>
</section>
<!-- One -->
<section id="one" class="wrapper style3 fullscreen fade-up">
<div class="inner">
<h2>📖 Tutorials</h2>
<section>
<h3>K2 Tutorials:</h3>
<div class="gallery-container" id="galleryContainer1">
<a id="prevButton1" class="button nav prevButton">‹</a>
<div class="gallery-wrapper" id="galleryWrapper1">
<div class="gallery" id="gallery1">
<div style="justify-content: center">
<img src="images/dataprep_k2.jpeg" alt="Image 1">
<h4>Data Prep (coming soon)</h4>
<h4><a href="https://github.com/LLM360/k2-data-prep" target="_blank">Code available on GitHub</a></h4>
</div>
<div>
<img src="images/traning_k2.jpeg" alt="Image 1">
<h4>Training (coming soon)</h4>
<h4><a href="https://github.com/LLM360/k2-train" target="_blank">Code available on GitHub</a></h4>
</div>
<div>
<img src="images/evaluation_k2.jpeg" alt="Image 1">
<h4>Evaluation (coming soon)</h4>
</div>
<!-- <div>-->
<!-- <img src="images/k2_analysis.jpeg" alt="Image 1">-->
<!-- <h4>Analysis (coming soon)</h4>-->
<!-- </div>-->
</div>
</div>
<a id="nextButton1" class="button nav nextButton">›</a>
</div>
</section>
<section>
<h3>Crystal Tutorials:</h3>
<div class="gallery-container" id="galleryContainer2">
<a id="prevButton2" class="button nav prevButton">‹</a>
<div class="gallery-wrapper" id="galleryWrapper2">
<div class="gallery" id="gallery2">
<div style="justify-content: center">
<img src="images/dataprep_crystal.jpeg" alt="Image 1">
<h4>Data Prep (coming soon)</h4>
<h4><a href="https://github.com/LLM360/crystalcoder-data-prep" target="_blank">Code available on GitHub</a></h4>
</div>
<div>
<img src="images/training_crystal.jpeg" alt="Image 1">
<h4>Training (coming soon)</h4>
<h4><a href="https://github.com/LLM360/crystalcoder-train" target="_blank">Code available on GitHub</a></h4>
</div>
<div>
<img src="images/evaluation_crystal.jpeg" alt="Image 1">
<h4>Evaluation (coming soon)</h4>
</div>
<!-- <div>-->
<!-- <img src="images/analysis_crystal.jpeg" alt="Image 1">-->
<!-- <h4>Analysis (coming soon)</h4>-->
<!-- </div>-->
</div>
</div>
<a id="nextButton2" class="button nav nextButton">›</a>
</div>
</section>
<section>
<h3>Amber Tutorials:</h3>
<div class="gallery-container" id="galleryContainer3">
<a id="prevButton3" class="button nav prevButton">‹</a>
<div class="gallery-wrapper" id="galleryWrapper3">
<div class="gallery" id="gallery3">
<div style="justify-content: center">
<img src="images/dataprep_amber.jpeg" alt="Image 1">
<h4>Data Prep (coming soon)</h4>
<h4><a href="https://github.com/LLM360/amber-data-prep" target="_blank">Code available on GitHub</a></h4>
</div>
<div>
<img src="images/training_amber.jpeg" alt="Image 1">
<h4>Training (coming soon)</h4>
<h4><a href="https://github.com/LLM360/amber-train" target="_blank">Code available on GitHub</a></h4>
</div>
<div>
<img src="images/evaluation_amber.jpeg" alt="Image 1">
<h4>Evaluation (coming soon)</h4>
</div>
<!-- <div>-->
<!-- <img src="images/analysis_amber.jpeg" alt="Image 1">-->
<!-- <h4>Analysis (coming soon)</h4>-->
<!-- </div>-->
</div>
</div>
<a id="nextButton3" class="button nav nextButton">›</a>
</div>
</section>
</div>
</section>
<!-- Two -->
<section id="two" class="wrapper fullscreen fade-up">
<div class="inner">
<h2>Pretraining Result References</h2>
<p>Please review the key results provided for your reference and check if your model's results are correct 🤗</p>
<table>
<thead>
<tr>
<th>Models</th>
<th>Check results</th>
</tr>
</thead>
<tbody>
<tr>
<td>K2</td>
<td><img src="images/k2_bg.png" style="max-width: 1.5em;margin-right: 0.5em"/><a href="https://huggingface.co/spaces/LLM360/k2-gallery" target="_blank">K2 Prompt Gallery</a></td>
</tr>
<tr>
<td>K2</td>
<td><img src="images/k2_bg.png" style="max-width: 1.5em;margin-right: 0.5em"/><a href="https://huggingface.co/spaces/LLM360/k2-eval-gallery" target="_blank">K2 Evaluation Gallery</a></td>
</tr>
<tr>
<td>K2</td>
<td><img src="images/wandb_logo.png" style="max-width: 1.5em;margin-right: 0.5em"/><a href="https://wandb.ai/llm360/K2?nw=29mu6l0zzqq" target="_blank">W&B</a></td>
</tr>
<tr>
<td>Crystal</td>
<td><img src="images/wandb_logo.png" style="max-width: 1.5em;margin-right: 0.5em"/><a href="https://wandb.ai/llm360/CrystalCoder?nw=hdze3lfpuer" target="_blank">W&B</a></td>
</tr>
<tr>
<td>Amber</td>
<td><img src="images/wandb_logo.png" style="max-width: 1.5em;margin-right: 0.5em"/><a href="https://wandb.ai/llm360/Amber/table?nw=lnzi8o2g4z" target="_blank">W&B</a></td>
</tr>
</tbody>
</table>
</div>
</section>
<!-- Three -->
<!-- <section id="three" class="wrapper fullscreen fade-up">-->
<!-- <div class="inner">-->
<!-- <h2>Pretraining Guide</h2>-->
<!-- <p>The LLM360 Pretraining Guide provides complete details and considerations for anyone leading a pre-training project. The Guide is divided into three categories: Project Planning, Initial Training Decision, Ensuring Success.Learning and procedural steps were gathered from the team while training Amber, Crystal, and K2 models. All training artifacts are made available to be reviewed and built off of.</p>-->
<!-- <div class="row aln-middle">-->
<!-- <div class="col-4 col-12-medium">-->
<!-- <span class="image main">-->
<!-- <img src="images/pretrainingguide.png"/>-->
<!-- </span>-->
<!-- </div>-->
<!-- <div class="col-8 col-12-medium">-->
<!-- <div class="features two">-->
<!-- <section>-->
<!-- <h3>Walkthrough of LLM Training</h3>-->
<!-- <ul>-->
<!-- <li>Goal and Budgets</li>-->
<!-- <li>Data Preparation</li>-->
<!-- <li>Model Architecture Choices</li>-->
<!-- <li>Hyperparameter study</li>-->
<!-- <li>Training Curriculum Planning</li>-->
<!-- <li>Preparing Runtime</li>-->
<!-- <li>The Training Job</li>-->
<!-- <li>Training Wrap Up</li>-->
<!-- </ul>-->
<!-- </section>-->
<!-- <section>-->
<!-- <h3>The LLM360 Open-source Initiative</h3>-->
<!-- <ul>-->
<!-- <li>What is LLM360 open source</li>-->
<!-- <li>LLM360 Case Studies</li>-->
<!-- </ul>-->
<!-- </section>-->
<!-- </div>-->
<!-- </div>-->
<!-- </div>-->
<!-- <form action="https://formspree.io/f/xzbnkzqg" method="POST" name="Pretraining Waitlist">-->
<!-- <label>Your email:</label>-->
<!-- <div class="fields">-->
<!-- <div class="field half">-->
<!-- <input type="email" name="demo-email" id="demo-email" value="" required placeholder="to be notified when it is available" />-->
<!-- </div>-->
<!-- </div>-->
<!-- <ul class="actions">-->
<!-- <li><a href="" class="button submit">Email Me</a></li>-->
<!-- </ul>-->
<!-- </form>-->
<!-- </div>-->
<!-- </section>-->
</div>
<!-- Footer -->
<footer id="footer" class="wrapper style1-alt">
<div class="inner">
<ul class="menu">
<p>
LLM360, proudly sponsored by Petuum and MBZUAI, is dedicated to advancing the field of AI by providing comprehensive access to large language models.<br>
LLM360 enables community-owned AGI by creating standards and tools to advance the bleeding edge of LLM capability and empower knowledge transfer, research, and development.
</p>
<ul class="actions">
<li><a href="https://twitter.com/llm360" target="_blank" class="icon brands circle fa-twitter"><span class="label">Twitter</span></a></li>
<li><a href="https://github.com/LLM360" target="_blank" class="icon brands circle fa-github"><span class="label">Github</span></a></li>
<li><a href="mailto:team@llm360.ai" target="_blank" class="icon circle fa-envelope"><span class="label">Email</span></a></li>
</ul>
<p class="copyright">© LLM360 2023-2024. All rights reserved.</p>
</ul>
</div>
</footer>
<!-- Scripts -->
<script src="assets/js/jquery.min.js"></script>
<script src="assets/js/jquery.scrollex.min.js"></script>
<script src="assets/js/jquery.scrolly.min.js"></script>
<script src="assets/js/browser.min.js"></script>
<script src="assets/js/breakpoints.min.js"></script>
<script src="assets/js/util.js"></script>
<script src="assets/js/main.js"></script>
</body>
</html></title>
</head>
<body>
</body>
</html>