-
Notifications
You must be signed in to change notification settings - Fork 0
/
pretrainingguide.html
232 lines (221 loc) · 10.5 KB
/
pretrainingguide.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
<!DOCTYPE HTML>
<html lang="en">
<head>
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-0FFN6N7318"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-0FFN6N7318');
</script>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no" />
<!-- SEO -->
<title>Pretraining Guide | LLM360</title>
<meta name="description" content="Delve into the essentials of LLM pre-training with the LLM360 Pre-Training Guide. This resource offers comprehensive insights into project planning, initial training decisions, and methods to ensure success, drawing on experiences from past projects like Amber and Crystal."/>
<link rel="canonical" href="https://www.llm360.ai/pretrainingguide.html" />
<meta name="keywords" content="LLM360, Open Source AI, Large Language Models, AI Research, Petuum, MBZUAI, Mohamed bin Zayed University of Artificial Intelligence, Amber LLM, CrystalCode, NLP, Language Processing"/>
<meta name="author" content="Petuum, Mohamed bin Zayed University of Artificial Intelligence"/>
<meta name="robots" content="index, follow"/>
<!-- Open Graph Protocol -->
<meta property="og:title" content="Pretraining Guide | LLM360"/>
<meta property="og:description" content="Delve into the essentials of LLM pre-training with the LLM360 Pre-Training Guide. This resource offers comprehensive insights into project planning, initial training decisions, and methods to ensure success, drawing on experiences from past projects like Amber and Crystal."/>
<meta property="og:type" content="website"/>
<meta property="og:url" content="https://www.llm360.ai/about.html/"/>
<meta property="og:image" content="https://www.llm360.ai/images/pic00.jpg"/>
<!-- Twitter Card -->
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:site" content="@llm360">
<meta name="twitter:title" content="Pretraining Guide | LLM360">
<meta name="twitter:description" content="Delve into the essentials of LLM pre-training with the LLM360 Pre-Training Guide. This resource offers comprehensive insights into project planning, initial training decisions, and methods to ensure success, drawing on experiences from past projects like Amber and Crystal.">
<meta name="twitter:image" content="https://www.llm360.ai/images/pic00.jpg">
<!-- Schema Markup -->
<script type="application/ld+json">
{
"@context": "http://schema.org",
"@type": "WebSite",
"name": "LLM360",
"url": "https://www.llm360.ai/about.html/",
"description": "Delve into the essentials of LLM pre-training with the LLM360 Pre-Training Guide. This resource offers comprehensive insights into project planning, initial training decisions, and methods to ensure success, drawing on experiences from past projects like Amber and Crystal.",
"publisher": {
"@type": "Organization",
"name": "Petuum, Mohamed bin Zayed University of Artificial Intelligence (MBZUAI)"
},
"image": {
"@type": "ImageObject",
"url": "https://www.llm360.ai/images/pic00.jpg",
"width": 1920,
"height": 1080
}
}
</script>
<!-- /SEO -->
<!-- Favicon -->
<link rel="icon" type="image/x-icon" href="assets/favicon/favicon.ico" />
<link rel="icon" type="image/png" sizes="192x192" href="assets/favicon/android-chrome-192x192.png">
<link rel="icon" type="image/png" sizes="512x512" href="assets/favicon/android-chrome-512x512.png">
<link rel="icon" type="image/png" sizes="32x32" href="assets/favicon/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="assets/favicon/favicon-16x16.png">
<link rel="apple-touch-icon" sizes="180x180" href="assets/favicon/apple-touch-icon.png">
<!-- <link rel="manifest" href="/site.webmanifest"> -->
<meta name="msapplication-TileColor" content="#da532c">
<meta name="theme-color" content="#ffffff">
<!-- Style -->
<link rel="stylesheet" href="assets/css/main.css" />
</head>
<body class="is-preload">
<!-- Wrapper -->
<div id="wrapper">
<!-- Main -->
<div id="main">
<div class="inner">
<!-- Guide -->
<section id="trainingguide" class="wrapper style7">
<div class="content">
<header class="major">
<h1>LLM360 Pre-Training Guide</h1>
<h3></h3>
</header>
<div class="row gtr-50">
<div class="col-3 col-12-large" style="padding-right: 1em">
<ul class="slider">
<li><img src="images/pt4.png" alt="" />
</li>
<li><img src="images/pt2.png" alt="" />
</li>
<li>
<img src="images/pt3.png" alt="" />
</li>
<li>
<img src="images/pt1.png" alt="" />
</li>
</ul>
</div>
<div class="col-9 col-12-large">
<h4>The LLM360 Pre-Training Guide provides complete details and considerations for anyone leading a pre-training project. The Guide is divided into three categories: Project Planning, Initial Training Decision, Ensuring Success.Learning and procedural steps were gathered from the team while training Amber, Crystal, and [65B] models. All training artifacts are made available to be reviewed and built off of.</h4>
<h5>Content:</h5>
<button class="collapsible">Walkthrough of LLM Training</button>
<div class="content_collapse">
<ul>
<li>Goal and Budgets</li>
<li>Data Preparation</li>
<li>Model Architecture Choices</li>
<li>Hyperparameter study</li>
<li>Training Curriculum Planning</li>
<li>Preparing Runtime</li>
<li>The Training Job</li>
<li>Training Wrap Up</li>
</ul>
</div>
<button class="collapsible">The LLM360 Open-source Initiative</button>
<div class="content_collapse">
<ul>
<li>What is LLM360 open source</li>
<li>LLM360 Case Studies</li>
</ul>
</div>
</div>
</div>
<div class="row aln-center" style="margin-top: 2em">
<div class="col-4 col-12-xsmall">
<form action="https://formspree.io/f/xzbnkzqg" method="POST" name="Pretraining Waitlist">
<label>Your email:</label>
<input type="email" name="demo-email" id="demo-email" value="" required placeholder="to be notified when it is available" />
<div class="col-12 mt-20">
<ul class="actions">
<li><input type="submit" value="Email Me" class="primary"></li>
</ul>
</div>
</form>
</div>
</div>
</div>
<header>
<h2>We find these resources valuable:</h2>
</header>
<div class="box secondary">
<div class="posts">
<article3>
<a href="https://blog.eleuther.ai/fm-dev-cheatsheet/" target="_blank" class="image"><img src="images/eleutheral.png" alt="" /></a>
<p>The Foundation Model Development Cheatsheet</p>
</article3>
<article3>
<a href="https://lmsys.org/blog/2024-03-01-policy/" target="_blank" class="image"><img src="images/lmsys.png" alt="" /></a>
<p>LMSYS Chatbot Arena: Live and Community-Driven LLM Evaluation</p>
</article3>
<article3>
<a href="https://blog.allenai.org/olmo-open-language-model-87ccfc95f580" target="_blank" class="image"><img src="images/olmo.png" alt="" /></a>
<p>OLMo: Open Language Model</p>
</article3>
<!-- <article3>-->
<!-- <a href="#" class="image"><img src="images/r4.png" alt="" /></a>-->
<!-- <p>Recourse4</p>-->
<!-- </article3>-->
<!-- <article3>-->
<!-- <a href="#" class="image"><img src="images/r1.png" alt="" /></a>-->
<!-- <p>Recourse5</p>-->
<!-- </article3>-->
<!-- <article3>-->
<!-- <a href="#" class="image"><img src="images/r2.png" alt="" /></a>-->
<!-- <p>Recourse6</p>-->
<!-- </article3>-->
<!-- <article3>-->
<!-- <a href="#" class="image"><img src="images/r3.png" alt="" /></a>-->
<!-- <p>Recourse7</p>-->
<!-- </article3>-->
<!-- <article3>-->
<!-- <a href="#" class="image"><img src="images/r4.png" alt="" /></a>-->
<!-- <p>Recourse8</p>-->
<!-- </article3>-->
</div>
</div>
</section>
</div>
</div>
<!-- Sidebar -->
<div id="sidebar">
<div class="inner">
<!-- Menu -->
<nav id="menu">
<a href="index.html#top">
<figure class="hover-rotate">
<img src="images/logo-highres.png" alt="logo" />
</figure>
</a>
<header>
<h1><a href="index.html#top">LLM360</a></h1>
</header>
<ul>
<li><a href="index.html#crystal">Models</a></li>
<li><a href="index.html#research">Research</a></li>
<li><a href="index.html#resources">Resources</a></li>
<li><a href="index.html#contact">Contact</a></li>
<li><a href="about.html">About</a></li>
</ul>
</nav>
<!-- Footer -->
<footer id="footer">
<p>
LLM360, proudly sponsored by Petuum, MBZUAI, and Cerebras, is dedicated to advancing the field of AI by providing comprehensive access to large language models.<br>
Our mission is to foster an ecosystem of collaboration, transparency, and innovation in AI research and applications.
</p>
<ul class="icons">
<li><a href="https://twitter.com/llm360" class="icon brands circle fa-twitter"><span class="label">Twitter</span></a></li>
<li><a href="https://github.com/LLM360" class="icon brands circle fa-github"><span class="label">Github</span></a></li>
<li><a href="mailto:[email protected]" class="icon circle fa-envelope"><span class="label">Email</span></a></li>
</ul>
<p class="copyright">© LLM360 2023. All rights reserved.</p>
</footer>
</div>
</div>
<a id="buttonToTop"></a>
</div>
<!-- Scripts -->
<script src="assets/js/jquery.min.js"></script>
<script src="assets/js/browser.min.js"></script>
<script src="assets/js/breakpoints.min.js"></script>
<script src="assets/js/util.js"></script>
<script src="assets/js/main.js"></script>
</body>
</html>