-
Notifications
You must be signed in to change notification settings - Fork 22
/
chapter4.html
368 lines (305 loc) · 16.8 KB
/
chapter4.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, shrink-to-fit=no">
<!-- <meta http-equiv="X-UA-Compatible" content="IE=edge"> -->
<!-- Social media -->
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:site" content="@OKAI" />
<!-- BB Change this--><meta name="twitter:title" content="Chapter 4: Perceptron Learning" />
<!-- CC Change this--><meta name="twitter:description" content="In this chapter, we are explaining how to make a perceptron learn to perform certain tasks." />
<!-- AA Change this--><meta name="twitter:image" content="https://okai.brown.edu/img/share/en/en_4.png" />
<!-- Change this--><meta property="og:url" content="https://okai.brown.edu/chapter4.html" />
<meta property="og:type" content="article" />
<!-- BB Change this--><meta property="og:title" content="Chapter 4: Perceptron Learning" />
<!-- CC Change this--><meta property="og:description" content="In this chapter, we are explaining how to make a perceptron learn to perform certain tasks." />
<!-- AA Change this--><meta property="og:image" content="https://okai.brown.edu/img/share/en/en_4.png" />
<!-- ................... -->
<!-- Google Fonts -->
<link href="https://fonts.googleapis.com/css?family=Nunito:400,700" rel="stylesheet">
<!-- Font Awesome -->
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.0/css/all.css" integrity="sha384-lZN37f5QGtY3VHgisS14W3ExzMWZxybE1SJSEsQp9S+oqd12jhcu+A56Ebc1zFSJ"
crossorigin="anonymous">
<!-- Bootstrap CSS CDN -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css"
integrity="sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4"
crossorigin="anonymous">
<!-- Scrollbar Custom CSS -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/malihu-custom-scrollbar-plugin/3.1.5/jquery.mCustomScrollbar.min.css">
<!-- My stylesheet -->
<link rel="stylesheet" href="./css/general.css">
<!-- change to own stylesheet -->
<link rel="stylesheet" href="./css/chapter4.css">
<!-- language annotations -->
<link rel="alternate" hreflang="zh-Hans" href="https://okai.brown.edu/zh/chapter4.html" />
<link rel="alternate" hreflang="en" href="https://okai.brown.edu/chapter4.html" />
<link rel="canonical" href="https://okai.brown.edu/chapter4.html" />
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-133914635-1"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag() {
dataLayer.push(arguments);
}
gtag('js', new Date());
gtag('config', 'UA-133914635-1');
</script>
<title>Chapter 4 - OKAI</title>
</head>
<body>
<!-- scrollbar -->
<div class="scrollBar" id="myBar"></div>
<!-- navbar place holder -->
<div id="nav-placeholder"></div>
<!-- ######################### -->
<!-- Chapter content beginning -->
<!-- ######################### -->
<!-- landing section -->
<div id="openingText" class="d-flex align-items-center justify-content-center">
<!-- start of row 1 -->
<div class="row align-items-center justify-content-center">
<!-- col 1 -->
<div class="d-none d-md-block col-1 arrowIcon" style="position: absolute; left: 5px;">
<a href="./chapter3.html" data-toggle="tooltip" title="Chapter 3"><i class="fas fa-caret-left fa-2x"></i></a>
</div>
<!-- col 2 -->
<div class="col-10 col-md-4 order-2 order-md-1 text-center">
<div class="text-left">
<p>Chapter 4</p>
<h2>Perceptron Learning</h2>
<p>How do we make a perceptron learn to perform certain tasks? In this chapter,
we are going behind the scenes of the “Perceptron Learning Institute”. Keep
scrolling!
</p>
</div>
</div>
<!-- col 3 -->
<div class="col-md-5 order-1 order-md-2">
<div id="openingLottie" class="openingAnimation mx-auto"></div>
</div>
<!-- col-4 -->
<div class="d-none d-md-block col-1 arrowIcon" style="position: absolute; right: 5px;">
<a href="./chapter5.html" data-toggle="tooltip" title="Chapter 5"><i class="fas fa-caret-right fa-2x"></i></a>
</div>
</div>
<!-- end of row 1 -->
<!-- start of row 2 -->
<div class="" style="position: absolute; bottom: 20px;">
<div>
<p class="iconHeight" id="scrollIcon">Scroll</p>
<i class="iconHeight fas fa-angle-down fa-lg"></i>
</div>
<!-- end of row 2 -->
</div>
</div>
<!-- end of landing section -->
<!-- ######## Animation & text beginning ######## -->
<!-- animation divs -->
<div id="functionLottie" class="animation animLeft upperLayer"></div>
<div id="lossLottie" class="animation animRight topLayer"></div>
<div id="gradientLottie" class="animation animLeft threeLayer"></div>
<!-- text begins -->
<!-- function animation text section -->
<div id="text1" class="bg-1grey scene">
<div class="text right">
<p>In the last chapter, we learned that a perceptron can take in some inputs and spit
out an output. How can we make it learn to perform tasks? To understand the
learning process, we will first discuss what the perceptron is abstractly.
</p>
</div>
</div>
<div id="text2" class="bg-1grey scene">
<div class="text right">
<p>At a high level, the perceptron can be seen as a mathematical function that takes in
some inputs, some weights, and some biases, and mixes them together to generate an
output.
</p>
</div>
</div>
<div id="text3" class="bg-1grey scene">
<div class="text right">
<p>Since we have no control over the inputs (we cannot change the data that we are
learning from), we will try to change weights and bias so that the perceptron
function will produce an ideal output for all the input we stuffed into it.
</p>
</div>
</div>
<!-- loss animation text section -->
<div id="text4" class="scene bg-2grey upperLayer">
<div class="text left">
<p>The core of “training” a perceptron is to let it understand how far its actions or
predictions are from the ground truth (desired results). To achieve this, we use a
<strong class="highlight" data-toggle="tooltip" title="A function that determines the loss,
the value that represents how far the output predictions are from the
ground truth.">loss
function</strong> to measure the performance of our perceptron. A commonly used
loss function is called cross entropy, which is very good at tasks like classifying
images. An ideal training algorithm modifies the weights and biases so the loss
decreases.
</p>
</div>
</div>
<div id="text5" class="scene bg-2grey upperLayer">
<div class="text left">
<p>
Here, the loss is visualized as the difference between the output (triangle) and
the
desired ground truth output (rectangle). Training the perceptron is the process of
decreasing this difference. Once the loss is decreased to its
<strong class="highlight" data-toggle="tooltip" title="It
is often hard to decrease the loss to the absolute minimum (global minimum), but we
can always reach a local minimum (a minimum value within a region).">minimum</strong>,
the perceptron
completes its learning.
</p>
</div>
</div>
<div id="text6" class="scene bg-2grey upperLayer">
<div class="text left">
<p>There are many loss functions, but all share a common behavior: the further off the
prediction is from the ground truth, the larger the output of the loss function is.
Minimizing the loss by changing the weights and biases means our perceptron will
gradually improve until it reaches its limit.
</p>
</div>
</div>
<!-- gradient animation text section-->
<div id="text7" class="scene bg-3grey topLayer">
<div class="text right">
<p>To minimize loss, we use an <strong data-toggle="tooltip" class="highlight" title="A technique to minimize loss">optimizer</strong>
called
<strong data-toggle="tooltip" class="highlight" title="Gradient Descent can be
quite inefficient in practice, so a variant called the Stochastic Gradient Descent (SGD) is used.">
Gradient Descent</strong>.
The "hill" here depicts the <strong data-toggle="tooltip" class="highlight" title="The position
on the map corresponds to the weights and bias of the network. The height of
the hill represents its loss.">relationship</strong>
between the weights and bias of a perceptron and its loss. The higher up on the
hill we are, the greater the loss is. Gradient Descent is a technique that modifies
the weights and biases to walk “downhill”, and the optimizer finds the steepest way
downwards.
</p>
</div>
</div>
<div id="text8" class="scene bg-3grey topLayer">
<div class="text right">
<p>
First, the Gradient Descent optimizer would look at the "hill" with respect to the
data, and generate <strong data-toggle="tooltip" class="highlight" title="A group of numbers indicating which
direction would be the “steepest” way downhill.">gradients</strong>,
providing a direction downhill. Then, the optimizer picks a learning rate to decide
how far to walk before stopping and redetermining the gradients. After many
repetitions, it slips its way down to a place where it can no longer go downhill
(minimum loss).
</p>
</div>
</div>
<!-- ending section -->
<div id="endingText" class="lastScene row align-items-center justify-content-around fiveLayer">
<!-- col 1 -->
<div class="d-none d-md-block col-1 arrowIcon" style="position: absolute; left: 5px;">
<a href="./chapter3.html" data-toggle="tooltip" title="Chapter 3"><i class="fas fa-caret-left fa-2x"></i></a>
</div>
<div class="col-12 col-md-10">
<div class="row justify-content-around">
<div class="col-10 col-md-5 fourLayer align-self-start">
<div class="text-left align-self-start">
<h3><strong>Summary</strong></h3>
<p>By now, we have learned what it means for the perceptron to learn, and
how we can train the perceptron. Let us venture into more complex
neural network structures. We will discuss how we can adapt perceptrons
to a type of multilayer networks called the feedforward neural network.
</p>
<h3><strong>Further Reading</strong></h3>
<p>
<a href="https://youtu.be/IHZwWFHWa-w" target="_blank">
Gradient descent, how neural networks learn </a>
<br><a href="https://towardsdatascience.com/gradient-descent-in-a-nutshell-eaf8c18212f0"
target="_blank">
Gradient Descent in a Nutshell</a>
<!-- <br><a href="https://cs.stanford.edu/people/eroberts/courses/soco/projects/neural-networks/Architecture/feedforward.html" target="_blank">
Neural Networks - Architecture - Stanford CS</a> -->
</p>
</div>
</div>
<div class="col-10 col-md-5 fourLayer align-self-start">
<div class="text-left">
<h3><strong>Glossary</strong></h3>
<p>
<strong>Loss Function</strong>: A function that determines the loss,
the value that represents how far the output predictions are from the
ground truth.
<br><br>
<strong>Gradient</strong>: A group of numbers indicating which
direction would be the “steepest” way to minimize loss (downhill).
<br><br>
<strong>Gradient Descent</strong>: An optimizer that uses gradients to
adjust the weights and biases to minimize loss.
<br><br>
</p>
</div>
</div>
<!-- mobile nav button groups -->
<div class="col-10 d-md-none fourLayer justify-content-start my-3">
<div class="btn-group d-flex" id="nextButtonGroup" role="group">
<a class="btn btn-warning w-50" href="./chapter3.html"
role="button" id="prevButton">Prev Chapter</a>
<span class="btn-warning btn-separator"></span>
<a class="btn btn-warning w-50" href="./chapter5.html"
role="button" id="nextButton">Next Chapter</a>
</div>
</div>
<!-- end of inner row -->
</div>
<!-- end of outer col-10 -->
</div>
<!-- col-4 -->
<div class="d-none d-md-block col-1 arrowIcon" style="position: absolute; right: 5px;">
<a href="./chapter5.html" data-toggle="tooltip" title="Chapter 5"><i class="fas fa-caret-right fa-2x"></i></a>
</div>
<!-- end of ending section -->
</div>
<!-- end of text area -->
<div class="overlay"></div>
<!-- svg goo filter DO NOT DELETE-->
<!-- <svg version="1.1" xmlns="https://www.w3.org/2000/svg" style="height: 0; width: 0; position: absolute; top: -9999px; left: -99990px">
<defs>
<filter color-interpolation-filters="sRGB" id="goo2">
<feGaussianBlur in="SourceGraphic" stdDeviation="8" result="blur" />
<feColorMatrix in="blur" mode="matrix" values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 21 -9" result="cm" />
<feBlend />
</filter>
</defs>
</svg> -->
<!-- Boostrap JavaScript -->
<script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.0/umd/popper.min.js"
integrity="sha384-cs/chFZiN24E4KMATLdqdvsezGxaGsi4hLGOzlXwp5UZB1LY//20VyM2taTB4QvJ"
crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/js/bootstrap.min.js" integrity="sha384-uefMccjFJAIv6A+rW+L4AHf99KvxDjWSu1z9VI8SKNVmz4sk7buKt/6v9KI65qnm"
crossorigin="anonymous"></script>
<!-- jQuery Custom Scroller CDN -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/malihu-custom-scrollbar-plugin/3.1.5/jquery.mCustomScrollbar.concat.min.js"></script>
<!-- GSAP -->
<script type="text/javascript" src="./js/TweenMax.min.js"></script>
<script type="text/javascript" src="./js/DrawSVGPlugin.min.js"></script>
<script type="text/javascript" src="./js/MorphSVGPlugin.min.js"></script>
<!-- Lottie -->
<script type="text/javascript" src="./js/lottie.js"></script>
<!-- ScrollMagic JavaScript -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/ScrollMagic/2.0.6/ScrollMagic.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/ScrollMagic/2.0.5/plugins/animation.gsap.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/ScrollMagic/2.0.5//plugins/debug.addIndicators.js"></script>
<script type="text/javascript" src="./js/main.js"></script>
<!-- change to own js files -->
<script type="text/javascript" src="./js/chapter4.js"></script>
<script type="text/javascript">
$(document).ready(function() {
$('[data-toggle="tooltip"]').tooltip({
placement: "top"
});
});
</script>
</body>
</html>