1. 模型简介
(1) 模型要能够学习恒等变换
(2) 隐藏层的神经元的活动性满足稀疏性(模型大脑)
(3) 隐藏层的权重矩阵同样满足稀疏性(一种正则化方法，实际上，在稀疏编码取得好的效果有启发作用)
2. 模型实现
# Sparse Auto-Encoder
#
# Author: HSW
# Date: 2018-05-07
#
import tensorflow as tf
import numpy as np
def axvier_init(fan_in, fan_out, constant = 1):
''' Initial weights '''
low = -constant + np.sqrt(6.0 / (fan_in + fan_out))
high = constant + np.sqrt(6.0 / (fan_in + fan_out))
return tf.random_uniform((fan_in, fan_out), minval = low, maxval = high, dtype = tf.float32)
class SparseAutoEncoder(object):
def __init__(self, AutoEncoder_Shape = (128, 256, 128), transfer_function = tf.nn.softplus, optimizer = tf.train.AdamOptimizer(), p = 0.05, sparse_beta = 0.1, weights_lambda = 0.00001):
''' Constructor Funcion '''
self.LayerCnt = len(AutoEncoder_Shape)
self.AutoEncoder_Shape = AutoEncoder_Shape
self.transfer = transfer_function
self.p = p
self.sparse_beta = sparse_beta
self.weights_lambda = weights_lambda
self.weights = self.init_weights()
self.layers = self.init_layers()
self.total_loss = self.init_loss()
self.optimizer = optimizer.minimize(self.total_loss)
init = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init)
def init_weights(self):
''' AutoEncoder Weights '''
all_weights = dict()
for iter in range(self.LayerCnt - 1):
weightName = "weight" + str(iter + 1)
biasName = "bias" + str(iter + 1)
if iter == self.LayerCnt - 2:
all_weights[weightName] = tf.Variable(axvier_init(self.AutoEncoder_Shape[iter], self.AutoEncoder_Shape[iter + 1]))
all_weights[biasName] = tf.Variable(tf.zeros([self.AutoEncoder_Shape[iter + 1]]))
else:
all_weights[weightName] = tf.Variable(tf.zeros([self.AutoEncoder_Shape[iter], self.AutoEncoder_Shape[iter + 1]]))
all_weights[biasName] = tf.Variable(tf.zeros([self.AutoEncoder_Shape[iter + 1]]))
return all_weights
def init_layers(self):
''' Sparse AutoEncoder: Input Layer + Hidden Layer + Output Layer '''
all_layers = dict()
all_layers['inputs'] = tf.placeholder(tf.float32, [None, self.AutoEncoder_Shape[0]])
for iter in range(self.LayerCnt - 1):
if iter == 0:
# inputs Layer => 1th Hidden Layer
preLayerName = 'inputs'
layerName = 'hidden' + str(iter + 1)
weightName = 'weight' + str(iter + 1)
biasName = 'bias' + str(iter + 1)
all_layers[layerName] = self.transfer(tf.add(tf.matmul(all_layers[preLayerName], self.weights[weightName]), self.weights[biasName]))
elif iter == self.LayerCnt - 2:
# Last Hidden Layer => outputs Layer
preLayerName = 'hidden' + str(iter)
layerName = 'outputs'
weightName = 'weight' + str(iter + 1)
biasName = 'bias' + str(iter + 1)
all_layers[layerName] = tf.add(tf.matmul(all_layers[preLayerName], self.weights[weightName]), self.weights[biasName])
else:
# pre-Hidden Layer => cur-Hidden Layer
preLayerName = 'hidden' + str(iter)
layerName = 'hidden' + str(iter + 1)
weightName = 'weight' + str(iter + 1)
biasName = 'bias' + str(iter + 1)
all_layers[layerName] = self.transfer(tf.add(tf.matmul(all_layers[preLayerName], self.weights[weightName]), self.weights[biasName]))
return all_layers
def init_loss(self):
''' Sparse AutoEncoder Loss '''
# data loss
data_loss = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.layers['outputs'], self.layers['inputs']), 2))
# sparse loss
active_loss = 0
weights_loss = 0
for iter in range(self.LayerCnt - 1):
weightName = 'weight' + str(iter + 1)
weights_loss += tf.norm(self.weights[weightName], ord='fro', axis=[-2, -1])
for iter in range(self.LayerCnt - 2):
hiddenName = 'hidden' + str(iter + 1)
sparse_loss = tf.reduce_mean(self.layers[hiddenName], 0)
active_loss += sparse_loss * tf.log(sparse_loss / self.p) + (1 - sparse_loss) * tf.log((1 - self.p) / (1 - sparse_loss))
return data_loss + self.sparse_beta * active_loss + self.weights_lambda * weights_loss
def partial_fit(self, inputs):
''' For train '''
cost, opt = self.sess.run((self.total_loss, self.optimizer), feed_dict={self.layers['inputs']: inputs})
return cost
def calc_total_cost(self, inputs):
''' For evalution '''
return self.sess.run(self.total_loss, feed_dict={self.inputs: inputs})
def transform(self, inputs, layerNum):
''' inputs => Encode Feature '''
hiddenName = 'hidden' + str(layerNum)
return self.sess.run(self.layers[hiddenName], feed_dict={self.layers['inputs']: inputs})
def generate(self, hidden = None, layerNum = 1):
''' Encode Feature => outputs '''
hiddenName = 'hidden' + str(layerNum)
return self.sess.run(self.layers['outputs'], feed_dict={self.layers[hiddenName]: hidden})
def reconstruct(self, inputs):
''' inputs => outputs '''
return self.sess.run(self.layers['outputs'], feed_dict={self.layers['inputs']: inputs})
def getWeigths(self, layerNum):
''' weight of layerNum-th layer '''
weightName = 'weight' + str(layerNum)
return self.sess.run(self.weights[weightName])
def getBiases(self, layerNum):
''' bias of layerNum-th layer '''
biasName = 'bias' + str(layerNum)
return self.sess.run(self.weights[biasName])
if __name__ == "__main__":
sparseAutoEncoder = SparseAutoEncoder();
print(sparseAutoEncoder.layers)
print(sparseAutoEncoder.weights['weight1'])
print(sparseAutoEncoder.weights['bias1'])
print(sparseAutoEncoder.weights['weight2'])
print(sparseAutoEncoder.weights['bias2'])
3. 模型测试
版权声明：本文为hit1524468原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接和本声明。