Our manual backprop weight average is missing about deeplearningzerotoall HOT 12 CLOSED

N = X.shape[0]
d_W1 = tf.matmul(tf.transpose(X), d_l1) / N

W1 = tf.Variable(tf.random_normal([2, 2]), name='weight1')
b1 = tf.Variable(tf.random_normal([2]), name='bias1')
layer1 = tf.sigmoid(tf.matmul(X, W1) + b1)

W2 = tf.Variable(tf.random_normal([2, 1]), name='weight2')
b2 = tf.Variable(tf.random_normal([1]), name='bias2')
Y_pred = tf.sigmoid(tf.matmul(layer1, W2) + b2)

# cost/loss function
cost = -tf.reduce_mean(Y * tf.log(Y_pred) + (1 - Y) *
                       tf.log(1 - Y_pred))

d_Y_pred = (Y_pred - Y) / (Y_pred * (1.0 - Y_pred) + 1e-7)
d_sigma = Y_pred * (1 - Y_pred)

# Layer 2
d_o2 = d_Y_pred * d_sigma
d_l2 = tf.multiply(d_o2, d_sigma)
d_b2 = d_l2
d_W2 = tf.matmul(tf.transpose(layer1), d_l2)

# Mean
d_b2_mean = tf.reduce_mean(d_b2, axis=[0])
d_W2_mean = d_W2 / tf.cast(tf.shape(layer1)[0], dtype=tf.float32)

# Layer 1
d_o1 = layer1 * (1-layer1)
d_l1 = tf.multiply(tf.matmul(d_l2, tf.transpose(W2)), d_o1)
d_b1 = d_l1
d_W1 = tf.matmul(tf.transpose(X), d_l1)

# Mean
d_W1_mean = d_W1 / tf.cast(tf.shape(X)[0], dtype=tf.float32)
d_b1_mean = tf.reduce_mean(d_b1, axis=[0])

# Weight update
step = [
  tf.assign(W2, W2 - learning_rate * d_W2_mean),
  tf.assign(b2, b2 - learning_rate * d_b2_mean),
  tf.assign(W1, W1 - learning_rate * d_W1_mean),
  tf.assign(b1, b1 - learning_rate * d_b1_mean)
]

# cost/loss function                                                          
cost = -tf.reduce_mean(Y * tf.log(Y_pred) + (1 - Y) *                         
                       tf.log(1 - Y_pred))                                    
                                                                              
# Loss derivative                                                             
d_Y_pred = (Y_pred - Y) / (Y_pred * (1.0 - Y_pred) + 1e-7)                    
                                                                              
# Layer 2                                                                     
d_sigma2 = Y_pred * (1 - Y_pred)                                              
d_l2 = d_Y_pred * d_sigma2                                                    
d_b2 = d_l2                                                                   
d_W2 = tf.matmul(tf.transpose(layer1), d_l2)                                  
                                                                              
# Mean                                                                        
d_b2_mean = tf.reduce_mean(d_b2, axis=[0])                                    
d_W2_mean = d_W2 / tf.cast(tf.shape(layer1)[0], dtype=tf.float32)             
                                                                              
# Layer 1                                                                     
d_sigma1 = layer1 * (1-layer1)                                                
d_l1 = d_l2 * d_sigma1                                                        
d_b1 = d_l1                                                                   
d_W1 = tf.matmul(tf.transpose(X), d_l1)                                       
                                                                              
# Mean                                                                        
d_W1_mean = d_W1 / tf.cast(tf.shape(X)[0], dtype=tf.float32)                  
d_b1_mean = tf.reduce_mean(d_b1, axis=[0])                                    
                                                                              
# Weight update                                                               
step = [                                                                      
  tf.assign(W2, W2 - learning_rate * d_W2_mean),                              
  tf.assign(b2, b2 - learning_rate * d_b2_mean),                              
  tf.assign(W1, W1 - learning_rate * d_W1_mean),                              
  tf.assign(b1, b1 - learning_rate * d_b1_mean)                               
]                                                                             

# Network                                                              
#          p1     a1           l1     p2     a2           l2 (y_pred)  
# X -> (*) -> (+) -> (sigmoid) -> (*) -> (+) -> (sigmoid) -> (loss)    
#       ^      ^                   ^      ^                            
#       |      |                   |      |                            
#       W1     b1                  W2     b2                           
                                                                       
# Loss derivative                                                      
d_Y_pred = (Y_pred - Y) / (Y_pred * (1.0 - Y_pred) + 1e-7)             
                                                                       
# Layer 2                                                              
d_sigma2 = Y_pred * (1 - Y_pred)                                       
d_a2 = d_Y_pred * d_sigma2                                             
d_p2 = d_a2                                                            
d_b2 = d_a2                                                            
d_W2 = tf.matmul(tf.transpose(l1), d_p2)                               
                                                                       
# Mean                                                                 
d_b2_mean = tf.reduce_mean(d_b2, axis=[0])                             
d_W2_mean = d_W2 / tf.cast(tf.shape(l1)[0], dtype=tf.float32)          
                                                                       
# Layer 1                                                              
d_l1 = tf.matmul(d_p2, tf.transpose(W2))                               
d_sigma1 = l1 * (1 - l1)                                               
d_a1 = d_l1 * d_sigma1                                                 
d_b1 = d_a1                                                            
d_p1 = d_a1                                                            
d_W1 = tf.matmul(tf.transpose(X), d_a1)                                
                                                                       
# Mean                                                                 
d_W1_mean = d_W1 / tf.cast(tf.shape(X)[0], dtype=tf.float32)           
d_b1_mean = tf.reduce_mean(d_b1, axis=[0])                             
                                                                       
# Weight update                                                        
step = [                                                               
  tf.assign(W2, W2 - learning_rate * d_W2_mean),                       
  tf.assign(b2, b2 - learning_rate * d_b2_mean),                       
  tf.assign(W1, W1 - learning_rate * d_W1_mean),                       
  tf.assign(b1, b1 - learning_rate * d_b1_mean)                        
]