.. code:: python
%matplotlib inline
from d2l import mxnet as d2l
import math
from mxnet import np, npx
npx.set_np()
def adagrad_2d(x1, x2, s1, s2):
eps = 1e-6
g1, g2 = 0.2 * x1, 4 * x2
s1 += g1 ** 2
s2 += g2 ** 2
x1 -= eta / math.sqrt(s1 + eps) * g1
x2 -= eta / math.sqrt(s2 + eps) * g2
return x1, x2, s1, s2
def f_2d(x1, x2):
return 0.1 * x1 ** 2 + 2 * x2 ** 2
eta = 0.4
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
.. figure:: output_adagrad_2fb0ed_3_0.svg
.. raw:: html

.. raw:: html
.. code:: python
%matplotlib inline
from d2l import torch as d2l
import math
import torch
def adagrad_2d(x1, x2, s1, s2):
eps = 1e-6
g1, g2 = 0.2 * x1, 4 * x2
s1 += g1 ** 2
s2 += g2 ** 2
x1 -= eta / math.sqrt(s1 + eps) * g1
x2 -= eta / math.sqrt(s2 + eps) * g2
return x1, x2, s1, s2
def f_2d(x1, x2):
return 0.1 * x1 ** 2 + 2 * x2 ** 2
eta = 0.4
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
.. figure:: output_adagrad_2fb0ed_6_0.svg
.. raw:: html

.. raw:: html
.. code:: python
%matplotlib inline
from d2l import tensorflow as d2l
import math
import tensorflow as tf
def adagrad_2d(x1, x2, s1, s2):
eps = 1e-6
g1, g2 = 0.2 * x1, 4 * x2
s1 += g1 ** 2
s2 += g2 ** 2
x1 -= eta / math.sqrt(s1 + eps) * g1
x2 -= eta / math.sqrt(s2 + eps) * g2
return x1, x2, s1, s2
def f_2d(x1, x2):
return 0.1 * x1 ** 2 + 2 * x2 ** 2
eta = 0.4
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
.. figure:: output_adagrad_2fb0ed_9_0.svg
.. raw:: html

.. raw:: html
.. code:: python
eta = 2
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
.. figure:: output_adagrad_2fb0ed_15_0.svg
.. raw:: html

.. raw:: html
.. code:: python
eta = 2
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
.. figure:: output_adagrad_2fb0ed_18_0.svg
.. raw:: html

.. raw:: html
.. code:: python
eta = 2
d2l.show_trace_2d(f_2d, d2l.train_2d(adagrad_2d))
.. figure:: output_adagrad_2fb0ed_21_0.svg
.. raw:: html

.. raw:: html
.. code:: python
def init_adagrad_states(feature_dim):
s_w = np.zeros((feature_dim, 1))
s_b = np.zeros(1)
return (s_w, s_b)
def adagrad(params, states, hyperparams):
eps = 1e-6
for p, s in zip(params, states):
s[:] += np.square(p.grad)
p[:] -= hyperparams['lr'] * p.grad / np.sqrt(s + eps)
.. raw:: html

.. raw:: html
.. code:: python
def init_adagrad_states(feature_dim):
s_w = torch.zeros((feature_dim, 1))
s_b = torch.zeros(1)
return (s_w, s_b)
def adagrad(params, states, hyperparams):
eps = 1e-6
for p, s in zip(params, states):
with torch.no_grad():
s[:] += torch.square(p.grad)
p[:] -= hyperparams['lr'] * p.grad / torch.sqrt(s + eps)
p.grad.data.zero_()
.. raw:: html

.. raw:: html
.. code:: python
def init_adagrad_states(feature_dim):
s_w = tf.Variable(tf.zeros((feature_dim, 1)))
s_b = tf.Variable(tf.zeros(1))
return (s_w, s_b)
def adagrad(params, grads, states, hyperparams):
eps = 1e-6
for p, s, g in zip(params, states, grads):
s[:].assign(s + tf.math.square(g))
p[:].assign(p - hyperparams['lr'] * g / tf.math.sqrt(s + eps))
.. raw:: html

.. raw:: html
.. code:: python
data_iter, feature_dim = d2l.get_data_ch11(batch_size=10)
d2l.train_ch11(adagrad, init_adagrad_states(feature_dim),
{'lr': 0.1}, data_iter, feature_dim);
.. parsed-literal::
:class: output
loss: 0.242, 0.076 sec/epoch
.. figure:: output_adagrad_2fb0ed_39_1.svg
.. raw:: html

.. raw:: html
.. code:: python
data_iter, feature_dim = d2l.get_data_ch11(batch_size=10)
d2l.train_ch11(adagrad, init_adagrad_states(feature_dim),
{'lr': 0.1}, data_iter, feature_dim);
.. parsed-literal::
:class: output
loss: 0.242, 0.012 sec/epoch
.. figure:: output_adagrad_2fb0ed_42_1.svg
.. raw:: html

.. raw:: html
.. code:: python
data_iter, feature_dim = d2l.get_data_ch11(batch_size=10)
d2l.train_ch11(adagrad, init_adagrad_states(feature_dim),
{'lr': 0.1}, data_iter, feature_dim);
.. parsed-literal::
:class: output
loss: 0.253, 0.119 sec/epoch
.. figure:: output_adagrad_2fb0ed_45_1.svg
.. raw:: html

.. raw:: html
.. code:: python
d2l.train_concise_ch11('adagrad', {'learning_rate': 0.1}, data_iter)
.. parsed-literal::
:class: output
loss: 0.242, 0.086 sec/epoch
.. figure:: output_adagrad_2fb0ed_51_1.svg
.. raw:: html

.. raw:: html
.. code:: python
trainer = torch.optim.Adagrad
d2l.train_concise_ch11(trainer, {'lr': 0.1}, data_iter)
.. parsed-literal::
:class: output
loss: 0.242, 0.011 sec/epoch
.. figure:: output_adagrad_2fb0ed_54_1.svg
.. raw:: html

.. raw:: html
.. code:: python
trainer = tf.keras.optimizers.Adagrad
d2l.train_concise_ch11(trainer, {'learning_rate' : 0.1}, data_iter)
.. parsed-literal::
:class: output
loss: 0.242, 0.093 sec/epoch
.. figure:: output_adagrad_2fb0ed_57_1.svg
.. raw:: html

.. raw:: html
`Discussions `__
.. raw:: html

.. raw:: html
`Discussions `__
.. raw:: html

.. raw:: html
`Discussions `__
.. raw:: html

.. raw:: html