.. _sec_rnn_scratch:
Thực hiện các mạng nơ-ron tái phát từ đầu
=========================================
Trong phần này, chúng tôi sẽ triển khai RNN từ đầu cho mô hình ngôn ngữ
cấp ký tự, theo mô tả của chúng tôi trong :numref:`sec_rnn`. Một mô
hình như vậy sẽ được đào tạo trên H Gwells' \* The Time Machine\*. Như
trước đây, chúng ta bắt đầu bằng cách đọc tập dữ liệu đầu tiên, được
giới thiệu trong :numref:`sec_language_model`.
.. raw:: html
.. raw:: html
.. code:: python
%matplotlib inline
import math
from mxnet import autograd, gluon, np, npx
from d2l import mxnet as d2l
npx.set_np()
batch_size, num_steps = 32, 35
train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)
.. raw:: html
.. raw:: html
.. code:: python
%matplotlib inline
import math
import torch
from torch import nn
from torch.nn import functional as F
from d2l import torch as d2l
batch_size, num_steps = 32, 35
train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)
.. raw:: html
.. raw:: html
.. code:: python
%matplotlib inline
import math
import tensorflow as tf
from d2l import tensorflow as d2l
batch_size, num_steps = 32, 35
train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)
train_random_iter, vocab_random_iter = d2l.load_data_time_machine(
batch_size, num_steps, use_random_iter=True)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
npx.one_hot(np.array([0, 2]), len(vocab))
.. parsed-literal::
:class: output
array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])
.. raw:: html
.. raw:: html
.. code:: python
F.one_hot(torch.tensor([0, 2]), len(vocab))
.. parsed-literal::
:class: output
tensor([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0],
[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0]])
.. raw:: html
.. raw:: html
.. code:: python
tf.one_hot(tf.constant([0, 2]), len(vocab))
.. parsed-literal::
:class: output
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
X = np.arange(10).reshape((2, 5))
npx.one_hot(X.T, 28).shape
.. parsed-literal::
:class: output
(5, 2, 28)
.. raw:: html
.. raw:: html
.. code:: python
X = torch.arange(10).reshape((2, 5))
F.one_hot(X.T, 28).shape
.. parsed-literal::
:class: output
torch.Size([5, 2, 28])
.. raw:: html
.. raw:: html
.. code:: python
X = tf.reshape(tf.range(10), (2, 5))
tf.one_hot(tf.transpose(X), 28).shape
.. parsed-literal::
:class: output
TensorShape([5, 2, 28])
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def get_params(vocab_size, num_hiddens, device):
num_inputs = num_outputs = vocab_size
def normal(shape):
return np.random.normal(scale=0.01, size=shape, ctx=device)
# Hidden layer parameters
W_xh = normal((num_inputs, num_hiddens))
W_hh = normal((num_hiddens, num_hiddens))
b_h = np.zeros(num_hiddens, ctx=device)
# Output layer parameters
W_hq = normal((num_hiddens, num_outputs))
b_q = np.zeros(num_outputs, ctx=device)
# Attach gradients
params = [W_xh, W_hh, b_h, W_hq, b_q]
for param in params:
param.attach_grad()
return params
.. raw:: html
.. raw:: html
.. code:: python
def get_params(vocab_size, num_hiddens, device):
num_inputs = num_outputs = vocab_size
def normal(shape):
return torch.randn(size=shape, device=device) * 0.01
# Hidden layer parameters
W_xh = normal((num_inputs, num_hiddens))
W_hh = normal((num_hiddens, num_hiddens))
b_h = torch.zeros(num_hiddens, device=device)
# Output layer parameters
W_hq = normal((num_hiddens, num_outputs))
b_q = torch.zeros(num_outputs, device=device)
# Attach gradients
params = [W_xh, W_hh, b_h, W_hq, b_q]
for param in params:
param.requires_grad_(True)
return params
.. raw:: html
.. raw:: html
.. code:: python
def get_params(vocab_size, num_hiddens):
num_inputs = num_outputs = vocab_size
def normal(shape):
return tf.random.normal(shape=shape,stddev=0.01,mean=0,dtype=tf.float32)
# Hidden layer parameters
W_xh = tf.Variable(normal((num_inputs, num_hiddens)), dtype=tf.float32)
W_hh = tf.Variable(normal((num_hiddens, num_hiddens)), dtype=tf.float32)
b_h = tf.Variable(tf.zeros(num_hiddens), dtype=tf.float32)
# Output layer parameters
W_hq = tf.Variable(normal((num_hiddens, num_outputs)), dtype=tf.float32)
b_q = tf.Variable(tf.zeros(num_outputs), dtype=tf.float32)
params = [W_xh, W_hh, b_h, W_hq, b_q]
return params
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def init_rnn_state(batch_size, num_hiddens, device):
return (np.zeros((batch_size, num_hiddens), ctx=device), )
.. raw:: html
.. raw:: html
.. code:: python
def init_rnn_state(batch_size, num_hiddens, device):
return (torch.zeros((batch_size, num_hiddens), device=device), )
.. raw:: html
.. raw:: html
.. code:: python
def init_rnn_state(batch_size, num_hiddens):
return (tf.zeros((batch_size, num_hiddens)), )
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def rnn(inputs, state, params):
# Shape of `inputs`: (`num_steps`, `batch_size`, `vocab_size`)
W_xh, W_hh, b_h, W_hq, b_q = params
H, = state
outputs = []
# Shape of `X`: (`batch_size`, `vocab_size`)
for X in inputs:
H = np.tanh(np.dot(X, W_xh) + np.dot(H, W_hh) + b_h)
Y = np.dot(H, W_hq) + b_q
outputs.append(Y)
return np.concatenate(outputs, axis=0), (H,)
.. raw:: html
.. raw:: html
.. code:: python
def rnn(inputs, state, params):
# Here `inputs` shape: (`num_steps`, `batch_size`, `vocab_size`)
W_xh, W_hh, b_h, W_hq, b_q = params
H, = state
outputs = []
# Shape of `X`: (`batch_size`, `vocab_size`)
for X in inputs:
H = torch.tanh(torch.mm(X, W_xh) + torch.mm(H, W_hh) + b_h)
Y = torch.mm(H, W_hq) + b_q
outputs.append(Y)
return torch.cat(outputs, dim=0), (H,)
.. raw:: html
.. raw:: html
.. code:: python
def rnn(inputs, state, params):
# Here `inputs` shape: (`num_steps`, `batch_size`, `vocab_size`)
W_xh, W_hh, b_h, W_hq, b_q = params
H, = state
outputs = []
# Shape of `X`: (`batch_size`, `vocab_size`)
for X in inputs:
X = tf.reshape(X,[-1,W_xh.shape[0]])
H = tf.tanh(tf.matmul(X, W_xh) + tf.matmul(H, W_hh) + b_h)
Y = tf.matmul(H, W_hq) + b_q
outputs.append(Y)
return tf.concat(outputs, axis=0), (H,)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
class RNNModelScratch: #@save
"""An RNN Model implemented from scratch."""
def __init__(self, vocab_size, num_hiddens, device, get_params,
init_state, forward_fn):
self.vocab_size, self.num_hiddens = vocab_size, num_hiddens
self.params = get_params(vocab_size, num_hiddens, device)
self.init_state, self.forward_fn = init_state, forward_fn
def __call__(self, X, state):
X = npx.one_hot(X.T, self.vocab_size)
return self.forward_fn(X, state, self.params)
def begin_state(self, batch_size, ctx):
return self.init_state(batch_size, self.num_hiddens, ctx)
.. raw:: html
.. raw:: html
.. code:: python
class RNNModelScratch: #@save
"""A RNN Model implemented from scratch."""
def __init__(self, vocab_size, num_hiddens, device,
get_params, init_state, forward_fn):
self.vocab_size, self.num_hiddens = vocab_size, num_hiddens
self.params = get_params(vocab_size, num_hiddens, device)
self.init_state, self.forward_fn = init_state, forward_fn
def __call__(self, X, state):
X = F.one_hot(X.T, self.vocab_size).type(torch.float32)
return self.forward_fn(X, state, self.params)
def begin_state(self, batch_size, device):
return self.init_state(batch_size, self.num_hiddens, device)
.. raw:: html
.. raw:: html
.. code:: python
class RNNModelScratch: #@save
"""A RNN Model implemented from scratch."""
def __init__(self, vocab_size, num_hiddens,
init_state, forward_fn, get_params):
self.vocab_size, self.num_hiddens = vocab_size, num_hiddens
self.init_state, self.forward_fn = init_state, forward_fn
self.trainable_variables = get_params(vocab_size, num_hiddens)
def __call__(self, X, state):
X = tf.one_hot(tf.transpose(X), self.vocab_size)
X = tf.cast(X, tf.float32)
return self.forward_fn(X, state, self.trainable_variables)
def begin_state(self, batch_size, *args, **kwargs):
return self.init_state(batch_size, self.num_hiddens)
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
num_hiddens = 512
net = RNNModelScratch(len(vocab), num_hiddens, d2l.try_gpu(), get_params,
init_rnn_state, rnn)
state = net.begin_state(X.shape[0], d2l.try_gpu())
Y, new_state = net(X.as_in_context(d2l.try_gpu()), state)
Y.shape, len(new_state), new_state[0].shape
.. parsed-literal::
:class: output
((10, 28), 1, (2, 512))
.. raw:: html
.. raw:: html
.. code:: python
num_hiddens = 512
net = RNNModelScratch(len(vocab), num_hiddens, d2l.try_gpu(), get_params,
init_rnn_state, rnn)
state = net.begin_state(X.shape[0], d2l.try_gpu())
Y, new_state = net(X.to(d2l.try_gpu()), state)
Y.shape, len(new_state), new_state[0].shape
.. parsed-literal::
:class: output
(torch.Size([10, 28]), 1, torch.Size([2, 512]))
.. raw:: html
.. raw:: html
.. code:: python
# defining tensorflow training strategy
device_name = d2l.try_gpu()._device_name
strategy = tf.distribute.OneDeviceStrategy(device_name)
num_hiddens = 512
with strategy.scope():
net = RNNModelScratch(len(vocab), num_hiddens, init_rnn_state, rnn,
get_params)
state = net.begin_state(X.shape[0])
Y, new_state = net(X, state)
Y.shape, len(new_state), new_state[0].shape
.. parsed-literal::
:class: output
(TensorShape([10, 28]), 1, TensorShape([2, 512]))
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def predict_ch8(prefix, num_preds, net, vocab, device): #@save
"""Generate new characters following the `prefix`."""
state = net.begin_state(batch_size=1, ctx=device)
outputs = [vocab[prefix[0]]]
get_input = lambda: np.array([outputs[-1]], ctx=device).reshape((1, 1))
for y in prefix[1:]: # Warm-up period
_, state = net(get_input(), state)
outputs.append(vocab[y])
for _ in range(num_preds): # Predict `num_preds` steps
y, state = net(get_input(), state)
outputs.append(int(y.argmax(axis=1).reshape(1)))
return ''.join([vocab.idx_to_token[i] for i in outputs])
.. raw:: html
.. raw:: html
.. code:: python
def predict_ch8(prefix, num_preds, net, vocab, device): #@save
"""Generate new characters following the `prefix`."""
state = net.begin_state(batch_size=1, device=device)
outputs = [vocab[prefix[0]]]
get_input = lambda: torch.tensor([outputs[-1]], device=device).reshape((1, 1))
for y in prefix[1:]: # Warm-up period
_, state = net(get_input(), state)
outputs.append(vocab[y])
for _ in range(num_preds): # Predict `num_preds` steps
y, state = net(get_input(), state)
outputs.append(int(y.argmax(dim=1).reshape(1)))
return ''.join([vocab.idx_to_token[i] for i in outputs])
.. raw:: html
.. raw:: html
.. code:: python
def predict_ch8(prefix, num_preds, net, vocab): #@save
"""Generate new characters following the `prefix`."""
state = net.begin_state(batch_size=1, dtype=tf.float32)
outputs = [vocab[prefix[0]]]
get_input = lambda: tf.reshape(tf.constant([outputs[-1]]), (1, 1)).numpy()
for y in prefix[1:]: # Warm-up period
_, state = net(get_input(), state)
outputs.append(vocab[y])
for _ in range(num_preds): # Predict `num_preds` steps
y, state = net(get_input(), state)
outputs.append(int(y.numpy().argmax(axis=1).reshape(1)))
return ''.join([vocab.idx_to_token[i] for i in outputs])
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
predict_ch8('time traveller ', 10, net, vocab, d2l.try_gpu())
.. parsed-literal::
:class: output
'time traveller iiiiiiiiii'
.. raw:: html
.. raw:: html
.. code:: python
predict_ch8('time traveller ', 10, net, vocab, d2l.try_gpu())
.. parsed-literal::
:class: output
'time traveller mkpbqhnnnn'
.. raw:: html
.. raw:: html
.. code:: python
predict_ch8('time traveller ', 10, net, vocab)
.. parsed-literal::
:class: output
'time traveller oitvdhmvdh'
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def grad_clipping(net, theta): #@save
"""Clip the gradient."""
if isinstance(net, gluon.Block):
params = [p.data() for p in net.collect_params().values()]
else:
params = net.params
norm = math.sqrt(sum((p.grad ** 2).sum() for p in params))
if norm > theta:
for param in params:
param.grad[:] *= theta / norm
.. raw:: html
.. raw:: html
.. code:: python
def grad_clipping(net, theta): #@save
"""Clip the gradient."""
if isinstance(net, nn.Module):
params = [p for p in net.parameters() if p.requires_grad]
else:
params = net.params
norm = torch.sqrt(sum(torch.sum((p.grad ** 2)) for p in params))
if norm > theta:
for param in params:
param.grad[:] *= theta / norm
.. raw:: html
.. raw:: html
.. code:: python
def grad_clipping(grads, theta): #@save
"""Clip the gradient."""
theta = tf.constant(theta, dtype=tf.float32)
new_grad = []
for grad in grads:
if isinstance(grad, tf.IndexedSlices):
new_grad.append(tf.convert_to_tensor(grad))
else:
new_grad.append(grad)
norm = tf.math.sqrt(sum((tf.reduce_sum(grad ** 2)).numpy()
for grad in new_grad))
norm = tf.cast(norm, tf.float32)
if tf.greater(norm, theta):
for i, grad in enumerate(new_grad):
new_grad[i] = grad * theta / norm
else:
new_grad = new_grad
return new_grad
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
#@save
def train_epoch_ch8(net, train_iter, loss, updater, device, use_random_iter):
"""Train a model within one epoch (defined in Chapter 8)."""
state, timer = None, d2l.Timer()
metric = d2l.Accumulator(2) # Sum of training loss, no. of tokens
for X, Y in train_iter:
if state is None or use_random_iter:
# Initialize `state` when either it is the first iteration or
# using random sampling
state = net.begin_state(batch_size=X.shape[0], ctx=device)
else:
for s in state:
s.detach()
y = Y.T.reshape(-1)
X, y = X.as_in_ctx(device), y.as_in_ctx(device)
with autograd.record():
y_hat, state = net(X, state)
l = loss(y_hat, y).mean()
l.backward()
grad_clipping(net, 1)
updater(batch_size=1) # Since the `mean` function has been invoked
metric.add(l * d2l.size(y), d2l.size(y))
return math.exp(metric[0] / metric[1]), metric[1] / timer.stop()
.. raw:: html
.. raw:: html
.. code:: python
#@save
def train_epoch_ch8(net, train_iter, loss, updater, device, use_random_iter):
"""Train a net within one epoch (defined in Chapter 8)."""
state, timer = None, d2l.Timer()
metric = d2l.Accumulator(2) # Sum of training loss, no. of tokens
for X, Y in train_iter:
if state is None or use_random_iter:
# Initialize `state` when either it is the first iteration or
# using random sampling
state = net.begin_state(batch_size=X.shape[0], device=device)
else:
if isinstance(net, nn.Module) and not isinstance(state, tuple):
# `state` is a tensor for `nn.GRU`
state.detach_()
else:
# `state` is a tuple of tensors for `nn.LSTM` and
# for our custom scratch implementation
for s in state:
s.detach_()
y = Y.T.reshape(-1)
X, y = X.to(device), y.to(device)
y_hat, state = net(X, state)
l = loss(y_hat, y.long()).mean()
if isinstance(updater, torch.optim.Optimizer):
updater.zero_grad()
l.backward()
grad_clipping(net, 1)
updater.step()
else:
l.backward()
grad_clipping(net, 1)
# Since the `mean` function has been invoked
updater(batch_size=1)
metric.add(l * y.numel(), y.numel())
return math.exp(metric[0] / metric[1]), metric[1] / timer.stop()
.. raw:: html
.. raw:: html
.. code:: python
#@save
def train_epoch_ch8(net, train_iter, loss, updater, use_random_iter):
"""Train a model within one epoch (defined in Chapter 8)."""
state, timer = None, d2l.Timer()
metric = d2l.Accumulator(2) # Sum of training loss, no. of tokens
for X, Y in train_iter:
if state is None or use_random_iter:
# Initialize `state` when either it is the first iteration or
# using random sampling
state = net.begin_state(batch_size=X.shape[0], dtype=tf.float32)
with tf.GradientTape(persistent=True) as g:
y_hat, state = net(X, state)
y = tf.reshape(tf.transpose(Y), (-1))
l = loss(y, y_hat)
params = net.trainable_variables
grads = g.gradient(l, params)
grads = grad_clipping(grads, 1)
updater.apply_gradients(zip(grads, params))
# Keras loss by default returns the average loss in a batch
# l_sum = l * float(d2l.size(y)) if isinstance(
# loss, tf.keras.losses.Loss) else tf.reduce_sum(l)
metric.add(l * d2l.size(y), d2l.size(y))
return math.exp(metric[0] / metric[1]), metric[1] / timer.stop()
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def train_ch8(net, train_iter, vocab, lr, num_epochs, device, #@save
use_random_iter=False):
"""Train a model (defined in Chapter 8)."""
loss = gluon.loss.SoftmaxCrossEntropyLoss()
animator = d2l.Animator(xlabel='epoch', ylabel='perplexity',
legend=['train'], xlim=[10, num_epochs])
# Initialize
if isinstance(net, gluon.Block):
net.initialize(ctx=device, force_reinit=True,
init=init.Normal(0.01))
trainer = gluon.Trainer(net.collect_params(),
'sgd', {'learning_rate': lr})
updater = lambda batch_size: trainer.step(batch_size)
else:
updater = lambda batch_size: d2l.sgd(net.params, lr, batch_size)
predict = lambda prefix: predict_ch8(prefix, 50, net, vocab, device)
# Train and predict
for epoch in range(num_epochs):
ppl, speed = train_epoch_ch8(
net, train_iter, loss, updater, device, use_random_iter)
if (epoch + 1) % 10 == 0:
animator.add(epoch + 1, [ppl])
print(f'perplexity {ppl:.1f}, {speed:.1f} tokens/sec on {str(device)}')
print(predict('time traveller'))
print(predict('traveller'))
.. raw:: html
.. raw:: html
.. code:: python
#@save
def train_ch8(net, train_iter, vocab, lr, num_epochs, device,
use_random_iter=False):
"""Train a model (defined in Chapter 8)."""
loss = nn.CrossEntropyLoss()
animator = d2l.Animator(xlabel='epoch', ylabel='perplexity',
legend=['train'], xlim=[10, num_epochs])
# Initialize
if isinstance(net, nn.Module):
updater = torch.optim.SGD(net.parameters(), lr)
else:
updater = lambda batch_size: d2l.sgd(net.params, lr, batch_size)
predict = lambda prefix: predict_ch8(prefix, 50, net, vocab, device)
# Train and predict
for epoch in range(num_epochs):
ppl, speed = train_epoch_ch8(
net, train_iter, loss, updater, device, use_random_iter)
if (epoch + 1) % 10 == 0:
print(predict('time traveller'))
animator.add(epoch + 1, [ppl])
print(f'perplexity {ppl:.1f}, {speed:.1f} tokens/sec on {str(device)}')
print(predict('time traveller'))
print(predict('traveller'))
.. raw:: html
.. raw:: html
.. code:: python
#@save
def train_ch8(net, train_iter, vocab, lr, num_epochs, strategy,
use_random_iter=False):
"""Train a model (defined in Chapter 8)."""
with strategy.scope():
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
updater = tf.keras.optimizers.SGD(lr)
animator = d2l.Animator(xlabel='epoch', ylabel='perplexity',
legend=['train'], xlim=[10, num_epochs])
predict = lambda prefix: predict_ch8(prefix, 50, net, vocab)
# Train and predict
for epoch in range(num_epochs):
ppl, speed = train_epoch_ch8(net, train_iter, loss, updater,
use_random_iter)
if (epoch + 1) % 10 == 0:
print(predict('time traveller'))
animator.add(epoch + 1, [ppl])
device = d2l.try_gpu()._device_name
print(f'perplexity {ppl:.1f}, {speed:.1f} tokens/sec on {str(device)}')
print(predict('time traveller'))
print(predict('traveller'))
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
num_epochs, lr = 500, 1
train_ch8(net, train_iter, vocab, lr, num_epochs, d2l.try_gpu())
.. parsed-literal::
:class: output
perplexity 1.0, 32184.1 tokens/sec on gpu(0)
time traveller for so it will be convenient to speak of himwas e
traveller with a slight accession ofcheerfulness really thi
.. figure:: output_rnn-scratch_546c4d_159_1.svg
.. raw:: html
.. raw:: html
.. code:: python
num_epochs, lr = 500, 1
train_ch8(net, train_iter, vocab, lr, num_epochs, d2l.try_gpu())
.. parsed-literal::
:class: output
perplexity 1.0, 75622.3 tokens/sec on cuda:0
time traveller with a slight accession ofcheerfulness really thi
traveller with a slight accession ofcheerfulness really thi
.. figure:: output_rnn-scratch_546c4d_162_1.svg
.. raw:: html
.. raw:: html
.. code:: python
num_epochs, lr = 500, 1
train_ch8(net, train_iter, vocab, lr, num_epochs, strategy)
.. parsed-literal::
:class: output
perplexity 1.0, 10388.9 tokens/sec on /GPU:0
time travelleryou can show black is white by argument said filby
travelleryou can show black is white by argument said filby
.. figure:: output_rnn-scratch_546c4d_165_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
net = RNNModelScratch(len(vocab), num_hiddens, d2l.try_gpu(), get_params,
init_rnn_state, rnn)
train_ch8(net, train_iter, vocab, lr, num_epochs, d2l.try_gpu(),
use_random_iter=True)
.. parsed-literal::
:class: output
perplexity 1.5, 26859.8 tokens/sec on gpu(0)
time traveller proceeded anyreal body must have extension in fou
travellerit s against reason said filbywhy argument said fi
.. figure:: output_rnn-scratch_546c4d_171_1.svg
.. raw:: html
.. raw:: html
.. code:: python
net = RNNModelScratch(len(vocab), num_hiddens, d2l.try_gpu(), get_params,
init_rnn_state, rnn)
train_ch8(net, train_iter, vocab, lr, num_epochs, d2l.try_gpu(),
use_random_iter=True)
.. parsed-literal::
:class: output
perplexity 1.4, 78034.7 tokens/sec on cuda:0
time traveller smiledcrination is to be accepted is an absolutel
traveller after the pauserequired for the proper assimilati
.. figure:: output_rnn-scratch_546c4d_174_1.svg
.. raw:: html
.. raw:: html
.. code:: python
with strategy.scope():
net = RNNModelScratch(len(vocab), num_hiddens, init_rnn_state, rnn,
get_params)
train_ch8(net, train_iter, vocab_random_iter, lr, num_epochs, strategy,
use_random_iter=True)
.. parsed-literal::
:class: output
perplexity 1.5, 10678.1 tokens/sec on /GPU:0
time traveller proceeded anyreal body must have extension in fou
traveller but now you begin to seethe object of my investig
.. figure:: output_rnn-scratch_546c4d_177_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html