.. raw:: html
.. code:: python
X = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
X.sum(0, keepdims=True), X.sum(1, keepdims=True)
.. parsed-literal::
:class: output
(array([[5., 7., 9.]]),
array([[ 6.],
[15.]]))
.. raw:: html
.. raw:: html
.. code:: python
X = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
X.sum(0, keepdim=True), X.sum(1, keepdim=True)
.. parsed-literal::
:class: output
(tensor([[5., 7., 9.]]),
tensor([[ 6.],
[15.]]))
.. raw:: html
.. raw:: html
.. code:: python
X = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
tf.reduce_sum(X, 0, keepdims=True), tf.reduce_sum(X, 1, keepdims=True)
.. parsed-literal::
:class: output
(,
)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def softmax(X):
X_exp = np.exp(X)
partition = X_exp.sum(1, keepdims=True)
return X_exp / partition # The broadcasting mechanism is applied here
.. raw:: html
.. raw:: html
.. code:: python
def softmax(X):
X_exp = torch.exp(X)
partition = X_exp.sum(1, keepdim=True)
return X_exp / partition # The broadcasting mechanism is applied here
.. raw:: html
.. raw:: html
.. code:: python
def softmax(X):
X_exp = tf.exp(X)
partition = tf.reduce_sum(X_exp, 1, keepdims=True)
return X_exp / partition # The broadcasting mechanism is applied here
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
X = np.random.normal(0, 1, (2, 5))
X_prob = softmax(X)
X_prob, X_prob.sum(1)
.. parsed-literal::
:class: output
(array([[0.22376052, 0.06659239, 0.06583703, 0.29964197, 0.3441681 ],
[0.63209665, 0.03179282, 0.194987 , 0.09209415, 0.04902935]]),
array([1. , 0.99999994]))
.. raw:: html
.. raw:: html
.. code:: python
X = torch.normal(0, 1, (2, 5))
X_prob = softmax(X)
X_prob, X_prob.sum(1)
.. parsed-literal::
:class: output
(tensor([[0.2157, 0.0482, 0.0647, 0.4588, 0.2126],
[0.0683, 0.0946, 0.1220, 0.3124, 0.4028]]),
tensor([1.0000, 1.0000]))
.. raw:: html
.. raw:: html
.. code:: python
X = tf.random.normal((2, 5), 0, 1)
X_prob = softmax(X)
X_prob, tf.reduce_sum(X_prob, 1)
.. parsed-literal::
:class: output
(,
)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def net(X):
return softmax(np.dot(X.reshape((-1, W.shape[0])), W) + b)
.. raw:: html
.. raw:: html
.. code:: python
def net(X):
return softmax(torch.matmul(X.reshape((-1, W.shape[0])), W) + b)
.. raw:: html
.. raw:: html
.. code:: python
def net(X):
return softmax(tf.matmul(tf.reshape(X, (-1, W.shape[0])), W) + b)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
y = np.array([0, 2])
y_hat = np.array([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y_hat[[0, 1], y]
.. parsed-literal::
:class: output
array([0.1, 0.5])
.. raw:: html
.. raw:: html
.. code:: python
y = torch.tensor([0, 2])
y_hat = torch.tensor([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y_hat[[0, 1], y]
.. parsed-literal::
:class: output
tensor([0.1000, 0.5000])
.. raw:: html
.. raw:: html
.. code:: python
y_hat = tf.constant([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y = tf.constant([0, 2])
tf.boolean_mask(y_hat, tf.one_hot(y, depth=y_hat.shape[-1]))
.. parsed-literal::
:class: output
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def cross_entropy(y_hat, y):
return - np.log(y_hat[range(len(y_hat)), y])
cross_entropy(y_hat, y)
.. parsed-literal::
:class: output
array([2.3025851, 0.6931472])
.. raw:: html
.. raw:: html
.. code:: python
def cross_entropy(y_hat, y):
return - torch.log(y_hat[range(len(y_hat)), y])
cross_entropy(y_hat, y)
.. parsed-literal::
:class: output
tensor([2.3026, 0.6931])
.. raw:: html
.. raw:: html
.. code:: python
def cross_entropy(y_hat, y):
return -tf.math.log(tf.boolean_mask(
y_hat, tf.one_hot(y, depth=y_hat.shape[-1])))
cross_entropy(y_hat, y)
.. parsed-literal::
:class: output
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def accuracy(y_hat, y): #@save
"""Compute the number of correct predictions."""
if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
y_hat = y_hat.argmax(axis=1)
cmp = y_hat.astype(y.dtype) == y
return float(cmp.astype(y.dtype).sum())
.. raw:: html
.. raw:: html
.. code:: python
def accuracy(y_hat, y): #@save
"""Compute the number of correct predictions."""
if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
y_hat = y_hat.argmax(axis=1)
cmp = y_hat.type(y.dtype) == y
return float(cmp.type(y.dtype).sum())
.. raw:: html
.. raw:: html
.. code:: python
def accuracy(y_hat, y): #@save
"""Compute the number of correct predictions."""
if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
y_hat = tf.argmax(y_hat, axis=1)
cmp = tf.cast(y_hat, y.dtype) == y
return float(tf.reduce_sum(tf.cast(cmp, y.dtype)))
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
accuracy(y_hat, y) / len(y)
.. parsed-literal::
:class: output
0.5
.. raw:: html
.. raw:: html
.. code:: python
accuracy(y_hat, y) / len(y)
.. parsed-literal::
:class: output
0.5
.. raw:: html
.. raw:: html
.. code:: python
accuracy(y_hat, y) / len(y)
.. parsed-literal::
:class: output
0.5
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def evaluate_accuracy(net, data_iter): #@save
"""Compute the accuracy for a model on a dataset."""
metric = Accumulator(2) # No. of correct predictions, no. of predictions
for X, y in data_iter:
metric.add(accuracy(net(X), y), d2l.size(y))
return metric[0] / metric[1]
.. raw:: html
.. raw:: html
.. code:: python
def evaluate_accuracy(net, data_iter): #@save
"""Compute the accuracy for a model on a dataset."""
if isinstance(net, torch.nn.Module):
net.eval() # Set the model to evaluation mode
metric = Accumulator(2) # No. of correct predictions, no. of predictions
with torch.no_grad():
for X, y in data_iter:
metric.add(accuracy(net(X), y), y.numel())
return metric[0] / metric[1]
.. raw:: html
.. raw:: html
.. code:: python
def evaluate_accuracy(net, data_iter): #@save
"""Compute the accuracy for a model on a dataset."""
metric = Accumulator(2) # No. of correct predictions, no. of predictions
for X, y in data_iter:
metric.add(accuracy(net(X), y), d2l.size(y))
return metric[0] / metric[1]
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
evaluate_accuracy(net, test_iter)
.. parsed-literal::
:class: output
0.0811
.. raw:: html
.. raw:: html
.. code:: python
evaluate_accuracy(net, test_iter)
.. parsed-literal::
:class: output
0.1485
.. raw:: html
.. raw:: html
.. code:: python
evaluate_accuracy(net, test_iter)
.. parsed-literal::
:class: output
0.1334
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def train_epoch_ch3(net, train_iter, loss, updater): #@save
"""Train a model within one epoch (defined in Chapter 3)."""
# Sum of training loss, sum of training accuracy, no. of examples
metric = Accumulator(3)
if isinstance(updater, gluon.Trainer):
updater = updater.step
for X, y in train_iter:
# Compute gradients and update parameters
with autograd.record():
y_hat = net(X)
l = loss(y_hat, y)
l.backward()
updater(X.shape[0])
metric.add(float(l.sum()), accuracy(y_hat, y), y.size)
# Return training loss and training accuracy
return metric[0] / metric[2], metric[1] / metric[2]
.. raw:: html
.. raw:: html
.. code:: python
def train_epoch_ch3(net, train_iter, loss, updater): #@save
"""The training loop defined in Chapter 3."""
# Set the model to training mode
if isinstance(net, torch.nn.Module):
net.train()
# Sum of training loss, sum of training accuracy, no. of examples
metric = Accumulator(3)
for X, y in train_iter:
# Compute gradients and update parameters
y_hat = net(X)
l = loss(y_hat, y)
if isinstance(updater, torch.optim.Optimizer):
# Using PyTorch in-built optimizer & loss criterion
updater.zero_grad()
l.mean().backward()
updater.step()
else:
# Using custom built optimizer & loss criterion
l.sum().backward()
updater(X.shape[0])
metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())
# Return training loss and training accuracy
return metric[0] / metric[2], metric[1] / metric[2]
.. raw:: html
.. raw:: html
.. code:: python
def train_epoch_ch3(net, train_iter, loss, updater): #@save
"""The training loop defined in Chapter 3."""
# Sum of training loss, sum of training accuracy, no. of examples
metric = Accumulator(3)
for X, y in train_iter:
# Compute gradients and update parameters
with tf.GradientTape() as tape:
y_hat = net(X)
# Keras implementations for loss takes (labels, predictions)
# instead of (predictions, labels) that users might implement
# in this book, e.g. `cross_entropy` that we implemented above
if isinstance(loss, tf.keras.losses.Loss):
l = loss(y, y_hat)
else:
l = loss(y_hat, y)
if isinstance(updater, tf.keras.optimizers.Optimizer):
params = net.trainable_variables
grads = tape.gradient(l, params)
updater.apply_gradients(zip(grads, params))
else:
updater(X.shape[0], tape.gradient(l, updater.params))
# Keras loss by default returns the average loss in a batch
l_sum = l * float(tf.size(y)) if isinstance(
loss, tf.keras.losses.Loss) else tf.reduce_sum(l)
metric.add(l_sum, accuracy(y_hat, y), tf.size(y))
# Return training loss and training accuracy
return metric[0] / metric[2], metric[1] / metric[2]
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
lr = 0.1
def updater(batch_size):
return d2l.sgd([W, b], lr, batch_size)
.. raw:: html
.. raw:: html
.. code:: python
lr = 0.1
def updater(batch_size):
return d2l.sgd([W, b], lr, batch_size)
.. raw:: html
.. raw:: html
.. code:: python
class Updater(): #@save
"""For updating parameters using minibatch stochastic gradient descent."""
def __init__(self, params, lr):
self.params = params
self.lr = lr
def __call__(self, batch_size, grads):
d2l.sgd(self.params, grads, self.lr, batch_size)
updater = Updater([W, b], lr=0.1)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
num_epochs = 10
train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
.. figure:: output_softmax-regression-scratch_a48321_177_0.svg
.. raw:: html
.. raw:: html
.. code:: python
num_epochs = 10
train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
.. figure:: output_softmax-regression-scratch_a48321_180_0.svg
.. raw:: html
.. raw:: html
.. code:: python
num_epochs = 10
train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
.. figure:: output_softmax-regression-scratch_a48321_183_0.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def predict_ch3(net, test_iter, n=6): #@save
"""Predict labels (defined in Chapter 3)."""
for X, y in test_iter:
break
trues = d2l.get_fashion_mnist_labels(y)
preds = d2l.get_fashion_mnist_labels(net(X).argmax(axis=1))
titles = [true +'\n' + pred for true, pred in zip(trues, preds)]
d2l.show_images(
X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n])
predict_ch3(net, test_iter)
.. figure:: output_softmax-regression-scratch_a48321_189_0.svg
.. raw:: html
.. raw:: html
.. code:: python
def predict_ch3(net, test_iter, n=6): #@save
"""Predict labels (defined in Chapter 3)."""
for X, y in test_iter:
break
trues = d2l.get_fashion_mnist_labels(y)
preds = d2l.get_fashion_mnist_labels(net(X).argmax(axis=1))
titles = [true +'\n' + pred for true, pred in zip(trues, preds)]
d2l.show_images(
X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n])
predict_ch3(net, test_iter)
.. figure:: output_softmax-regression-scratch_a48321_192_0.svg
.. raw:: html
.. raw:: html
.. code:: python
def predict_ch3(net, test_iter, n=6): #@save
"""Predict labels (defined in Chapter 3)."""
for X, y in test_iter:
break
trues = d2l.get_fashion_mnist_labels(y)
preds = d2l.get_fashion_mnist_labels(tf.argmax(net(X), axis=1))
titles = [true +'\n' + pred for true, pred in zip(trues, preds)]
d2l.show_images(
tf.reshape(X[0:n], (n, 28, 28)), 1, n, titles=titles[0:n])
predict_ch3(net, test_iter)
.. figure:: output_softmax-regression-scratch_a48321_195_0.svg
.. raw:: html
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html