.. _sec_fcn:
Mạng kết nối hoàn toàn
======================
Như đã thảo luận trong :numref:`sec_semantic_segmentation`, phân đoạn
ngữ nghĩa phân loại hình ảnh ở mức pixel. Một mạng phức tạp hoàn toàn
(FCN) sử dụng mạng thần kinh phức tạp để chuyển đổi pixel hình ảnh thành
các lớp pixel :cite:`Long.Shelhamer.Darrell.2015`. Không giống như các
CNN mà chúng ta gặp phải trước đó để phân loại hình ảnh hoặc phát hiện
đối tượng, một mạng phức tạp hoàn toàn biến đổi chiều cao và chiều rộng
của bản đồ tính năng trung gian trở lại hình ảnh đầu vào: điều này đạt
được bằng lớp biến đổi được giới thiệu trong
:numref:`sec_transposed_conv`. Do đó, đầu ra phân loại và hình ảnh đầu
vào có sự tương ứng một-một ở mức pixel: kích thước kênh ở bất kỳ điểm
ảnh đầu ra nào giữ kết quả phân loại cho pixel đầu vào ở cùng một vị trí
không gian.
.. raw:: html
.. raw:: html
.. code:: python
%matplotlib inline
from mxnet import gluon, image, init, np, npx
from mxnet.gluon import nn
from d2l import mxnet as d2l
npx.set_np()
.. raw:: html
.. raw:: html
.. code:: python
%matplotlib inline
import torch
import torchvision
from torch import nn
from torch.nn import functional as F
from d2l import torch as d2l
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
pretrained_net = gluon.model_zoo.vision.resnet18_v2(pretrained=True)
pretrained_net.features[-3:], pretrained_net.output
.. parsed-literal::
:class: output
(HybridSequential(
(0): Activation(relu)
(1): GlobalAvgPool2D(size=(1, 1), stride=(1, 1), padding=(0, 0), ceil_mode=True, global_pool=True, pool_type=avg, layout=NCHW)
(2): Flatten
),
Dense(512 -> 1000, linear))
.. raw:: html
.. raw:: html
.. code:: python
pretrained_net = torchvision.models.resnet18(pretrained=True)
list(pretrained_net.children())[-3:]
.. parsed-literal::
:class: output
[Sequential(
(0): BasicBlock(
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(downsample): Sequential(
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(1): BasicBlock(
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(relu): ReLU(inplace=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
),
AdaptiveAvgPool2d(output_size=(1, 1)),
Linear(in_features=512, out_features=1000, bias=True)]
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
net = nn.HybridSequential()
for layer in pretrained_net.features[:-2]:
net.add(layer)
.. raw:: html
.. raw:: html
.. code:: python
net = nn.Sequential(*list(pretrained_net.children())[:-2])
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
X = np.random.uniform(size=(1, 3, 320, 480))
net(X).shape
.. parsed-literal::
:class: output
(1, 512, 10, 15)
.. raw:: html
.. raw:: html
.. code:: python
X = torch.rand(size=(1, 3, 320, 480))
net(X).shape
.. parsed-literal::
:class: output
torch.Size([1, 512, 10, 15])
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
num_classes = 21
net.add(nn.Conv2D(num_classes, kernel_size=1),
nn.Conv2DTranspose(
num_classes, kernel_size=64, padding=16, strides=32))
.. raw:: html
.. raw:: html
.. code:: python
num_classes = 21
net.add_module('final_conv', nn.Conv2d(512, num_classes, kernel_size=1))
net.add_module('transpose_conv', nn.ConvTranspose2d(num_classes, num_classes,
kernel_size=64, padding=16, stride=32))
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def bilinear_kernel(in_channels, out_channels, kernel_size):
factor = (kernel_size + 1) // 2
if kernel_size % 2 == 1:
center = factor - 1
else:
center = factor - 0.5
og = (np.arange(kernel_size).reshape(-1, 1),
np.arange(kernel_size).reshape(1, -1))
filt = (1 - np.abs(og[0] - center) / factor) * \
(1 - np.abs(og[1] - center) / factor)
weight = np.zeros((in_channels, out_channels, kernel_size, kernel_size))
weight[range(in_channels), range(out_channels), :, :] = filt
return np.array(weight)
.. raw:: html
.. raw:: html
.. code:: python
def bilinear_kernel(in_channels, out_channels, kernel_size):
factor = (kernel_size + 1) // 2
if kernel_size % 2 == 1:
center = factor - 1
else:
center = factor - 0.5
og = (torch.arange(kernel_size).reshape(-1, 1),
torch.arange(kernel_size).reshape(1, -1))
filt = (1 - torch.abs(og[0] - center) / factor) * \
(1 - torch.abs(og[1] - center) / factor)
weight = torch.zeros((in_channels, out_channels,
kernel_size, kernel_size))
weight[range(in_channels), range(out_channels), :, :] = filt
return weight
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
conv_trans = nn.Conv2DTranspose(3, kernel_size=4, padding=1, strides=2)
conv_trans.initialize(init.Constant(bilinear_kernel(3, 3, 4)))
.. raw:: html
.. raw:: html
.. code:: python
conv_trans = nn.ConvTranspose2d(3, 3, kernel_size=4, padding=1, stride=2,
bias=False)
conv_trans.weight.data.copy_(bilinear_kernel(3, 3, 4));
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
img = image.imread('../img/catdog.jpg')
X = np.expand_dims(img.astype('float32').transpose(2, 0, 1), axis=0) / 255
Y = conv_trans(X)
out_img = Y[0].transpose(1, 2, 0)
.. raw:: html
.. raw:: html
.. code:: python
img = torchvision.transforms.ToTensor()(d2l.Image.open('../img/catdog.jpg'))
X = img.unsqueeze(0)
Y = conv_trans(X)
out_img = Y[0].permute(1, 2, 0).detach()
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
d2l.set_figsize()
print('input image shape:', img.shape)
d2l.plt.imshow(img.asnumpy());
print('output image shape:', out_img.shape)
d2l.plt.imshow(out_img.asnumpy());
.. parsed-literal::
:class: output
input image shape: (561, 728, 3)
output image shape: (1122, 1456, 3)
.. figure:: output_fcn_ce3435_75_1.svg
.. raw:: html
.. raw:: html
.. code:: python
d2l.set_figsize()
print('input image shape:', img.permute(1, 2, 0).shape)
d2l.plt.imshow(img.permute(1, 2, 0));
print('output image shape:', out_img.shape)
d2l.plt.imshow(out_img);
.. parsed-literal::
:class: output
input image shape: torch.Size([561, 728, 3])
output image shape: torch.Size([1122, 1456, 3])
.. figure:: output_fcn_ce3435_78_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
W = bilinear_kernel(num_classes, num_classes, 64)
net[-1].initialize(init.Constant(W))
net[-2].initialize(init=init.Xavier())
.. raw:: html
.. raw:: html
.. code:: python
W = bilinear_kernel(num_classes, num_classes, 64)
net.transpose_conv.weight.data.copy_(W);
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
batch_size, crop_size = 32, (320, 480)
train_iter, test_iter = d2l.load_data_voc(batch_size, crop_size)
.. parsed-literal::
:class: output
Downloading ../data/VOCtrainval_11-May-2012.tar from http://d2l-data.s3-accelerate.amazonaws.com/VOCtrainval_11-May-2012.tar...
read 1114 examples
read 1078 examples
.. raw:: html
.. raw:: html
.. code:: python
batch_size, crop_size = 32, (320, 480)
train_iter, test_iter = d2l.load_data_voc(batch_size, crop_size)
.. parsed-literal::
:class: output
read 1114 examples
read 1078 examples
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
num_epochs, lr, wd, devices = 5, 0.1, 1e-3, d2l.try_all_gpus()
loss = gluon.loss.SoftmaxCrossEntropyLoss(axis=1)
net.collect_params().reset_ctx(devices)
trainer = gluon.Trainer(net.collect_params(), 'sgd',
{'learning_rate': lr, 'wd': wd})
d2l.train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs, devices)
.. parsed-literal::
:class: output
loss 0.324, train acc 0.893, test acc 0.858
162.9 examples/sec on [gpu(0), gpu(1)]
.. figure:: output_fcn_ce3435_102_1.svg
.. raw:: html
.. raw:: html
.. code:: python
def loss(inputs, targets):
return F.cross_entropy(inputs, targets, reduction='none').mean(1).mean(1)
num_epochs, lr, wd, devices = 5, 0.001, 1e-3, d2l.try_all_gpus()
trainer = torch.optim.SGD(net.parameters(), lr=lr, weight_decay=wd)
d2l.train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs, devices)
.. parsed-literal::
:class: output
loss 0.446, train acc 0.863, test acc 0.854
252.5 examples/sec on [device(type='cuda', index=0), device(type='cuda', index=1)]
.. figure:: output_fcn_ce3435_105_1.svg
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def predict(img):
X = test_iter._dataset.normalize_image(img)
X = np.expand_dims(X.transpose(2, 0, 1), axis=0)
pred = net(X.as_in_ctx(devices[0])).argmax(axis=1)
return pred.reshape(pred.shape[1], pred.shape[2])
.. raw:: html
.. raw:: html
.. code:: python
def predict(img):
X = test_iter.dataset.normalize_image(img).unsqueeze(0)
pred = net(X.to(devices[0])).argmax(dim=1)
return pred.reshape(pred.shape[1], pred.shape[2])
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
def label2image(pred):
colormap = np.array(d2l.VOC_COLORMAP, ctx=devices[0], dtype='uint8')
X = pred.astype('int32')
return colormap[X, :]
.. raw:: html
.. raw:: html
.. code:: python
def label2image(pred):
colormap = torch.tensor(d2l.VOC_COLORMAP, device=devices[0])
X = pred.long()
return colormap[X, :]
.. raw:: html
.. raw:: html
.. raw:: html
.. code:: python
voc_dir = d2l.download_extract('voc2012', 'VOCdevkit/VOC2012')
test_images, test_labels = d2l.read_voc_images(voc_dir, False)
n, imgs = 4, []
for i in range(n):
crop_rect = (0, 0, 480, 320)
X = image.fixed_crop(test_images[i], *crop_rect)
pred = label2image(predict(X))
imgs += [X, pred, image.fixed_crop(test_labels[i], *crop_rect)]
d2l.show_images(imgs[::3] + imgs[1::3] + imgs[2::3], 3, n, scale=2);
.. figure:: output_fcn_ce3435_129_0.svg
.. raw:: html
.. raw:: html
.. code:: python
voc_dir = d2l.download_extract('voc2012', 'VOCdevkit/VOC2012')
test_images, test_labels = d2l.read_voc_images(voc_dir, False)
n, imgs = 4, []
for i in range(n):
crop_rect = (0, 0, 320, 480)
X = torchvision.transforms.functional.crop(test_images[i], *crop_rect)
pred = label2image(predict(X))
imgs += [X.permute(1,2,0), pred.cpu(),
torchvision.transforms.functional.crop(
test_labels[i], *crop_rect).permute(1,2,0)]
d2l.show_images(imgs[::3] + imgs[1::3] + imgs[2::3], 3, n, scale=2);
.. figure:: output_fcn_ce3435_132_0.svg
.. raw:: html
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html
`Discussions `__
.. raw:: html
.. raw:: html