## MindSpore / docs .gitee-modal { width: 500px !important; }

Explore and code with more than 6 million developers，Free private repositories ！：）
apply_deep_probability_programming.md 29.49 KB
zhangyi authored 2021-03-25 16:26 . modify links for r1.2 branch.

# 深度概率编程

Ascend GPU 全流程 初级 中级 高级

## 概述

1. 介绍如何使用bnn_layers模块实现贝叶斯神经网络（Bayesian Neural Network, BNN）；
2. 介绍如何使用variational模块dpn模块实现变分自编码器（Variational AutoEncoder, VAE）；
3. 介绍如何使用transforms模块实现DNN（Deep Neural Network, DNN）一键转BNN；
4. 介绍如何使用toolbox模块实现不确定性估计。

## 使用贝叶斯神经网络

1. 处理MNIST数据集；
2. 定义贝叶斯LeNet网络；
3. 定义损失函数和优化器；
4. 加载数据集并进行训练。

### 定义贝叶斯神经网络

import mindspore.nn as nn
from mindspore.nn.probability import bnn_layers
import mindspore.ops as ops

class BNNLeNet5(nn.Cell):
"""
bayesian Lenet network

Args:
num_class (int): Num classes. Default: 10.

Returns:
Tensor, output tensor
Examples:
>>> BNNLeNet5(num_class=10)

"""
def __init__(self, num_class=10):
super(BNNLeNet5, self).__init__()
self.num_class = num_class
self.fc1 = bnn_layers.DenseReparam(16 * 5 * 5, 120)
self.fc2 = bnn_layers.DenseReparam(120, 84)
self.fc3 = bnn_layers.DenseReparam(84, self.num_class)
self.relu = nn.ReLU()
self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2)
self.flatten = nn.Flatten()
self.reshape = ops.Reshape()

def construct(self, x):
x = self.conv1(x)
x = self.relu(x)
x = self.max_pool2d(x)
x = self.conv2(x)
x = self.relu(x)
x = self.max_pool2d(x)
x = self.flatten(x)
x = self.fc1(x)
x = self.relu(x)
x = self.fc2(x)
x = self.relu(x)
x = self.fc3(x)
return x

### 定义损失函数和优化器

MindSpore中定义损失函数和优化器的代码样例如下：

# loss function definition
criterion = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")

# optimization definition
optimizer = AdamWeightDecay(params=network.trainable_params(), learning_rate=0.0001)

### 训练网络

net_with_loss = bnn_layers.WithBNNLossCell(network, criterion, dnn_factor=60000, bnn_factor=0.000001)
train_bnn_network = TrainOneStepCell(net_with_loss, optimizer)
train_bnn_network.set_train()

train_set = create_dataset('./mnist_data/train', 64, 1)
test_set = create_dataset('./mnist_data/test', 64, 1)

epoch = 10

for i in range(epoch):
train_loss, train_acc = train_model(train_bnn_network, network, train_set)

valid_acc = validate_model(network, test_set)

print('Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tvalidation Accuracy: {:.4f}'.
format(i, train_loss, train_acc, valid_acc))

def train_model(train_net, net, dataset):
accs = []
loss_sum = 0
for _, data in enumerate(dataset.create_dict_iterator()):
train_x = Tensor(data['image'].asnumpy().astype(np.float32))
label = Tensor(data['label'].asnumpy().astype(np.int32))
loss = train_net(train_x, label)
output = net(train_x)
log_output = ops.LogSoftmax(axis=1)(output)
acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy())
accs.append(acc)
loss_sum += loss.asnumpy()

loss_sum = loss_sum / len(accs)
acc_mean = np.mean(accs)
return loss_sum, acc_mean

def validate_model(net, dataset):
accs = []
for _, data in enumerate(dataset.create_dict_iterator()):
train_x = Tensor(data['image'].asnumpy().astype(np.float32))
label = Tensor(data['label'].asnumpy().astype(np.int32))
output = net(train_x)
log_output = ops.LogSoftmax(axis=1)(output)
acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy())
accs.append(acc)

acc_mean = np.mean(accs)
return acc_mean

## 使用变分自编码器

1. 定义变分自编码器；
2. 定义损失函数和优化器；
3. 处理数据；
4. 训练网络；
5. 生成新样本或重构输入样本。

### 定义变分自编码器

class Encoder(nn.Cell):
def __init__(self):
super(Encoder, self).__init__()
self.fc1 = nn.Dense(1024, 800)
self.fc2 = nn.Dense(800, 400)
self.relu = nn.ReLU()
self.flatten = nn.Flatten()

def construct(self, x):
x = self.flatten(x)
x = self.fc1(x)
x = self.relu(x)
x = self.fc2(x)
x = self.relu(x)
return x

class Decoder(nn.Cell):
def __init__(self):
super(Decoder, self).__init__()
self.fc1 = nn.Dense(400, 1024)
self.sigmoid = nn.Sigmoid()
self.reshape = ops.Reshape()

def construct(self, z):
z = self.fc1(z)
z = self.reshape(z, IMAGE_SHAPE)
z = self.sigmoid(z)
return z

encoder = Encoder()
decoder = Decoder()
vae = VAE(encoder, decoder, hidden_size=400, latent_size=20)

### 定义损失函数和优化器

# loss function definition
net_loss = ELBO(latent_prior='Normal', output_prior='Normal')

# optimization definition

net_with_loss = nn.WithLossCell(vae, net_loss)

### 训练网络

from mindspore.nn.probability.infer import SVI

vi = SVI(net_with_loss=net_with_loss, optimizer=optimizer)
vae = vi.run(train_dataset=ds_train, epochs=10)
trained_loss = vi.get_train_loss()

### 生成新样本或重构输入样本

IMAGE_SHAPE = (-1, 1, 32, 32)
generated_sample = vae.generate_sample(64, IMAGE_SHAPE)
for sample in ds_train.create_dict_iterator():
sample_x = Tensor(sample['image'].asnumpy(), dtype=mstype.float32)
reconstructed_sample = vae.reconstruct_sample(sample_x)

## DNN一键转换成BNN

1. 定义DNN模型；
2. 定义损失函数和优化器；
3. 实现功能一：转换整个模型；
4. 实现功能二：转换指定类型的层。

### 定义DNN模型

from mindspore.common.initializer import TruncatedNormal
import mindspore.nn as nn
import mindspore.ops as ops

def conv(in_channels, out_channels, kernel_size, stride=1, padding=0):
"""weight initial for conv layer"""
weight = weight_variable()
return nn.Conv2d(in_channels, out_channels,

def fc_with_initialize(input_channels, out_channels):
"""weight initial for fc layer"""
weight = weight_variable()
bias = weight_variable()
return nn.Dense(input_channels, out_channels, weight, bias)

def weight_variable():
"""weight initial"""
return TruncatedNormal(0.02)

class LeNet5(nn.Cell):
"""
Lenet network

Args:
num_class (int): Num classes. Default: 10.

Returns:
Tensor, output tensor
Examples:
>>> LeNet5(num_class=10)

"""
def __init__(self, num_class=10):
super(LeNet5, self).__init__()
self.num_class = num_class
self.conv1 = conv(1, 6, 5)
self.conv2 = conv(6, 16, 5)
self.fc1 = fc_with_initialize(16 * 5 * 5, 120)
self.fc2 = fc_with_initialize(120, 84)
self.fc3 = fc_with_initialize(84, self.num_class)
self.relu = nn.ReLU()
self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2)
self.flatten = nn.Flatten()
self.reshape = ops.Reshape()

def construct(self, x):
x = self.conv1(x)
x = self.relu(x)
x = self.max_pool2d(x)
x = self.conv2(x)
x = self.relu(x)
x = self.max_pool2d(x)
x = self.flatten(x)
x = self.fc1(x)
x = self.relu(x)
x = self.fc2(x)
x = self.relu(x)
x = self.fc3(x)
return x

LeNet的网络结构如下：

LeNet5
(conv1) Conv2dinput_channels=1, output_channels=6, kernel_size=(5, 5),stride=(1, 1),  pad_mode=valid, padding=0, dilation=(1, 1), group=1, has_bias=False
(conv2) Conv2dinput_channels=6, output_channels=16, kernel_size=(5, 5),stride=(1, 1),  pad_mode=valid, padding=0, dilation=(1, 1), group=1, has_bias=False
(fc1) Densein_channels=400, out_channels=120, weight=Parameter (name=fc1.weight), has_bias=True, bias=Parameter (name=fc1.bias)
(fc2) Densein_channels=120, out_channels=84, weight=Parameter (name=fc2.weight), has_bias=True, bias=Parameter (name=fc2.bias)
(fc3) Densein_channels=84, out_channels=10, weight=Parameter (name=fc3.weight), has_bias=True, bias=Parameter (name=fc3.bias)
(relu) ReLU
(flatten) Flatten

### 定义损失函数和优化器

network = LeNet5()

# loss function definition
criterion = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")

# optimization definition

net_with_loss = WithLossCell(network, criterion)
train_network = TrainOneStepCell(net_with_loss, optimizer)

### 实例化TransformToBNN

TransformToBNN__init__函数定义如下：

class TransformToBNN:
def __init__(self, trainable_dnn, dnn_factor=1, bnn_factor=1):
net_with_loss = trainable_dnn.network
self.optimizer = trainable_dnn.optimizer
self.backbone = net_with_loss.backbone_network
self.loss_fn = getattr(net_with_loss, "_loss_fn")
self.dnn_factor = dnn_factor
self.bnn_factor = bnn_factor
self.bnn_loss_file = None

from mindspore.nn.probability import transforms

bnn_transformer = transforms.TransformToBNN(train_network, 60000, 0.000001)

### 实现功能一：转换整个模型

transform_to_bnn_model方法可以将整个DNN模型转换为BNN模型。其定义如下:

    def transform_to_bnn_model(self,
get_dense_args=lambda dp: {"in_channels": dp.in_channels, "has_bias": dp.has_bias,
"out_channels": dp.out_channels, "activation": dp.activation},
get_conv_args=lambda dp: {"in_channels": dp.in_channels, "out_channels": dp.out_channels,
"stride": dp.stride, "has_bias": dp.has_bias,
"group": dp.group},
r"""
Transform the whole DNN model to BNN model, and wrap BNN model by TrainOneStepCell.

Args:
get_dense_args (function): The arguments gotten from the DNN full connection layer. Default: lambda dp:
{"in_channels": dp.in_channels, "out_channels": dp.out_channels, "has_bias": dp.has_bias}.
get_conv_args (function): The arguments gotten from the DNN convolutional layer. Default: lambda dp:
"kernel_size": dp.kernel_size, "stride": dp.stride, "has_bias": dp.has_bias}.
add_dense_args (dict): The new arguments added to BNN full connection layer. Default: {}.
add_conv_args (dict): The new arguments added to BNN convolutional layer. Default: {}.

Returns:
Cell, a trainable BNN model wrapped by TrainOneStepCell.
"""

train_bnn_network = bnn_transformer.transform_to_bnn_model()

LeNet5
(conv1) ConvReparam
in_channels=1, out_channels=6, kernel_size=(5, 5), stride=(1, 1),  pad_mode=valid, padding=0, dilation=(1, 1), group=1, weight_mean=Parameter (name=conv1.weight_posterior.mean), weight_std=Parameter (name=conv1.weight_posterior.untransformed_std), has_bias=False
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(conv2) ConvReparam
in_channels=6, out_channels=16, kernel_size=(5, 5), stride=(1, 1),  pad_mode=valid, padding=0, dilation=(1, 1), group=1, weight_mean=Parameter (name=conv2.weight_posterior.mean), weight_std=Parameter (name=conv2.weight_posterior.untransformed_std), has_bias=False
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(fc1) DenseReparam
in_channels=400, out_channels=120, weight_mean=Parameter (name=fc1.weight_posterior.mean), weight_std=Parameter (name=fc1.weight_posterior.untransformed_std), has_bias=True, bias_mean=Parameter (name=fc1.bias_posterior.mean), bias_std=Parameter (name=fc1.bias_posterior.untransformed_std)
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(bias_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(bias_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(fc2) DenseReparam
in_channels=120, out_channels=84, weight_mean=Parameter (name=fc2.weight_posterior.mean), weight_std=Parameter (name=fc2.weight_posterior.untransformed_std), has_bias=True, bias_mean=Parameter (name=fc2.bias_posterior.mean), bias_std=Parameter (name=fc2.bias_posterior.untransformed_std)
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(bias_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(bias_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(fc3) DenseReparam
in_channels=84, out_channels=10, weight_mean=Parameter (name=fc3.weight_posterior.mean), weight_std=Parameter (name=fc3.weight_posterior.untransformed_std), has_bias=True, bias_mean=Parameter (name=fc3.bias_posterior.mean), bias_std=Parameter (name=fc3.bias_posterior.untransformed_std)
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(bias_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(bias_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(relu) ReLU
(flatten) Flatten

### 实现功能二：转换指定类型的层

transform_to_bnn_layer方法可以将DNN模型中指定类型的层（nn.Dense或者nn.Conv2d）转换为对应的贝叶斯层。其定义如下:

 def transform_to_bnn_layer(self, dnn_layer, bnn_layer, get_args=None, add_args=None):
r"""
Transform a specific type of layers in DNN model to corresponding BNN layer.

Args:
dnn_layer_type (Cell): The type of DNN layer to be transformed to BNN layer. The optional values are
nn.Dense, nn.Conv2d.
bnn_layer_type (Cell): The type of BNN layer to be transformed to. The optional values are
DenseReparameterization, ConvReparameterization.
get_args (dict): The arguments gotten from the DNN layer. Default: None.

Returns:
Cell, a trainable model wrapped by TrainOneStepCell, whose sprcific type of layer is transformed to the corresponding bayesian layer.
"""

train_bnn_network = bnn_transformer.transform_to_bnn_layer(nn.Dense, bnn_layers.DenseReparam)

LeNet5
(conv1) Conv2dinput_channels=1, output_channels=6, kernel_size=(5, 5),stride=(1, 1),  pad_mode=valid, padding=0, dilation=(1, 1), group=1, has_bias=False
(conv2) Conv2dinput_channels=6, output_channels=16, kernel_size=(5, 5),stride=(1, 1),  pad_mode=valid, padding=0, dilation=(1, 1), group=1, has_bias=False
(fc1) DenseReparam
in_channels=400, out_channels=120, weight_mean=Parameter (name=fc1.weight_posterior.mean), weight_std=Parameter (name=fc1.weight_posterior.untransformed_std), has_bias=True, bias_mean=Parameter (name=fc1.bias_posterior.mean), bias_std=Parameter (name=fc1.bias_posterior.untransformed_std)
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(bias_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(bias_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(fc2) DenseReparam
in_channels=120, out_channels=84, weight_mean=Parameter (name=fc2.weight_posterior.mean), weight_std=Parameter (name=fc2.weight_posterior.untransformed_std), has_bias=True, bias_mean=Parameter (name=fc2.bias_posterior.mean), bias_std=Parameter (name=fc2.bias_posterior.untransformed_std)
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(bias_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(bias_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(fc3) DenseReparam
in_channels=84, out_channels=10, weight_mean=Parameter (name=fc3.weight_posterior.mean), weight_std=Parameter (name=fc3.weight_posterior.untransformed_std), has_bias=True, bias_mean=Parameter (name=fc3.bias_posterior.mean), bias_std=Parameter (name=fc3.bias_posterior.untransformed_std)
(weight_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(weight_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(bias_prior) NormalPrior
(normal) Normalmean = 0.0, standard deviation = 0.1

(bias_posterior) NormalPosterior
(normal) Normalbatch_shape = None

(relu) ReLU
(flatten) Flatten

## 使用不确定性估计工具箱

• 偶然不确定性（Aleatoric Uncertainty）：描述数据中的内在噪声，即无法避免的误差，这个现象不能通过增加采样数据来削弱。
• 认知不确定性（Epistemic Uncertainty）：模型自身对输入数据的估计可能因为训练不佳、训练数据不够等原因而不准确，可以通过增加训练数据等方式来缓解。

from mindspore.nn.probability.toolbox.uncertainty_evaluation import UncertaintyEvaluation

network = LeNet5()
# get train and eval dataset
ds_train = create_dataset('workspace/mnist/train')
ds_eval = create_dataset('workspace/mnist/test')
evaluation = UncertaintyEvaluation(model=network,
train_dataset=ds_train,
num_classes=10,
epochs=1,
epi_uncer_model_path=None,
ale_uncer_model_path=None,
save_model=False)
for eval_data in ds_eval.create_dict_iterator():
eval_data = Tensor(eval_data['image'].asnumpy(), mstype.float32)
epistemic_uncertainty = evaluation.eval_epistemic_uncertainty(eval_data)
aleatoric_uncertainty = evaluation.eval_aleatoric_uncertainty(eval_data)
1
https://gitee.com/mindspore/docs.git
git@gitee.com:mindspore/docs.git
mindspore
docs
docs
r1.2