Hi Pumpikano,
With your help i was able to run the code on MNIST dataset. i am trying to train the model on another dataset. i have tried to modified the code accordingly, i will be very thankful if you can help me to solve this issue.
The code is
matplotlib inline
import tensorflow as tf
import numpy as np
import cPickle as pkl
from sklearn.manifold import TSNE
from flip_gradient import flip_gradient
from utils import *
from numpy import genfromtxt
import pdb
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
Process MNIST
mnist_train = (mnist.train.images > 0).reshape(55000, 28, 28, 1).astype(np.uint8) * 255
print(mnist_train.shape)
mnist_train = np.concatenate([mnist_train, mnist_train, mnist_train], 3)
mnist_test = (mnist.test.images > 0).reshape(10000, 28, 28, 1).astype(np.uint8) * 255
mnist_test = np.concatenate([mnist_test, mnist_test, mnist_test], 3)
mnist_train = genfromtxt('iv_train.csv', delimiter=',')
mnist_test = genfromtxt('iv_train.csv', delimiter=',')
mnist_train_labels = genfromtxt('iv_train_label.csv', delimiter=',')
mnist_test_labels = genfromtxt('iv_train_label.csv', delimiter=',')
mnist_train = mnist_train.reshape(5119, 20, 20, 1)
mnist_train = np.concatenate([mnist_train, mnist_train, mnist_train], 3)
mnist_test = mnist_test.reshape(5119, 20, 20, 1)
mnist_test = np.concatenate([mnist_test, mnist_test, mnist_test], 3)
print 'Source train data size',mnist_train.shape
print 'Source test data size',mnist_train.shape
pdb.set_trace()
Load MNIST-M
mnistm = pkl.load(open('mnistm_data.pkl'))
mnistm_train = mnistm['train']
mnistm_test = mnistm['test']
mnistm_valid = mnistm['valid']
mnistm_train = genfromtxt('iv_test_half.csv', delimiter=',')
mnistm_test = genfromtxt('iv_test_half.csv', delimiter=',')
mnistm_valid = genfromtxt('iv_test_half.csv', delimiter=',')
mnistm_train = mnistm_train.reshape(9000, 20, 20, 1)
mnistm_train = np.concatenate([mnistm_train, mnistm_train, mnistm_train], 3)
mnistm_test = mnistm_test.reshape(9000, 20, 20, 1)
mnistm_test = np.concatenate([mnistm_test, mnistm_test, mnistm_test], 3)
mnistm_valid = mnistm_valid.reshape(9000, 20, 20, 1)
mnistm_valid = np.concatenate([mnistm_valid, mnistm_valid, mnistm_valid], 3)
print 'Source train data size',mnistm_train.shape
print 'Source test data size',mnistm_test.shape
print 'Source test data size',mnistm_valid.shape
pdb.set_trace()
Compute pixel mean for normalizing data
pixel_mean = np.vstack([mnist_train, mnistm_train]).mean((0, 1, 2))
Create a mixed dataset for TSNE visualization
num_test = 500
combined_test_imgs = np.vstack([mnist_test[:num_test], mnistm_test[:num_test]])
combined_test_labels = np.vstack([mnist_test_labels[:num_test], mnist_test_labels[:num_test]])
combined_test_domain = np.vstack([np.tile([1., 0.], [num_test, 1]),
np.tile([0., 1.], [num_test, 1])])
imshow_grid(mnist_train)
imshow_grid(mnistm_train)
batch_size = 12
class MNISTModel(object):
"""Simple MNIST domain adaptation model."""
def init(self):
self._build_model()
def _build_model(self):
self.X = tf.placeholder(tf.uint8, [None, 20, 20, 3])
self.y = tf.placeholder(tf.float32, [None, 20])
self.domain = tf.placeholder(tf.float32, [None, 2])
self.l = tf.placeholder(tf.float32, [])
self.train = tf.placeholder(tf.bool, [])
X_input = tf.cast(self.X, tf.float32) - pixel_mean
# CNN model for feature extraction
with tf.variable_scope('feature_extractor'):
W_conv0 = weight_variable([5, 5, 3, 32])
b_conv0 = bias_variable([32])
h_conv0 = tf.nn.relu(conv2d(X_input, W_conv0) + b_conv0)
h_pool0 = max_pool_2x2(h_conv0)
W_conv1 = weight_variable([5, 5, 32, 48])
b_conv1 = bias_variable([48])
h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
# The domain-invariant feature
self.feature = tf.reshape(h_pool1, [-1, 4*8*75])
# MLP for class prediction
with tf.variable_scope('label_predictor'):
# Switches to route target examples (second half of batch) differently
# depending on train or test mode.
all_features = lambda: self.feature
source_features = lambda: tf.slice(self.feature, [0, 0], [batch_size, -1])
classify_feats = tf.cond(self.train, source_features, all_features)
all_labels = lambda: self.y
source_labels = lambda: tf.slice(self.y, [0, 0], [batch_size, -1])
self.classify_labels = tf.cond(self.train, source_labels, all_labels)
W_fc0 = weight_variable([4 * 8 * 75, 100])
b_fc0 = bias_variable([100])
h_fc0 = tf.nn.relu(tf.matmul(classify_feats, W_fc0) + b_fc0)
W_fc1 = weight_variable([100, 100])
b_fc1 = bias_variable([100])
h_fc1 = tf.nn.relu(tf.matmul(h_fc0, W_fc1) + b_fc1)
W_fc2 = weight_variable([100, 20])
b_fc2 = bias_variable([20])
#logits = tf.placeholder(tf.float32, [12, 20])
logits = tf.matmul(h_fc1, W_fc2) + b_fc2
print 'logit_size', logits.get_shape()
self.pred = tf.nn.softmax(logits)
self.pred_loss = tf.nn.softmax_cross_entropy_with_logits(logits, self.classify_labels)
# Small MLP for domain prediction with adversarial loss
with tf.variable_scope('domain_predictor'):
# Flip the gradient when backpropagating through this operation
feat = flip_gradient(self.feature, self.l)
d_W_fc0 = weight_variable([4*8*75, 100])
d_b_fc0 = bias_variable([100])
d_h_fc0 = tf.nn.relu(tf.matmul(feat, d_W_fc0) + d_b_fc0)
d_W_fc1 = weight_variable([100, 2])
d_b_fc1 = bias_variable([2])
d_logits = tf.matmul(d_h_fc0, d_W_fc1) + d_b_fc1
self.domain_pred = tf.nn.softmax(d_logits)
self.domain_loss = tf.nn.softmax_cross_entropy_with_logits(d_logits, self.domain)
Build the model graph
graph = tf.get_default_graph()
with graph.as_default():
model = MNISTModel()
learning_rate = tf.placeholder(tf.float32, [])
pred_loss = tf.reduce_mean(model.pred_loss)
domain_loss = tf.reduce_mean(model.domain_loss)
total_loss = pred_loss + domain_loss
regular_train_op = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(pred_loss)
dann_train_op = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(total_loss)
# Evaluation
correct_label_pred = tf.equal(tf.argmax(model.classify_labels, 1), tf.argmax(model.pred, 1))
label_acc = tf.reduce_mean(tf.cast(correct_label_pred, tf.float32))
correct_domain_pred = tf.equal(tf.argmax(model.domain, 1), tf.argmax(model.domain_pred, 1))
domain_acc = tf.reduce_mean(tf.cast(correct_domain_pred, tf.float32))
Params
num_steps = 8600
def train_and_evaluate(training_mode, graph, model, verbose=False):
"""Helper to run the model with different training modes."""
with tf.Session(graph=graph) as sess:
tf.initialize_all_variables().run()
# Batch generators
gen_source_batch = batch_generator(
[mnist_train, mnist_train_labels], batch_size)
gen_target_batch = batch_generator(
[mnistm_train, mnist_train_labels], batch_size)
gen_source_only_batch = batch_generator(
[mnist_train, mnist_train_labels], batch_size)
gen_target_only_batch = batch_generator(
[mnistm_train, mnist_train_labels], batch_size)
domain_labels = np.vstack([np.tile([1., 0.], [batch_size, 1]),
np.tile([0., 1.], [batch_size, 1])])
# Training loop
for i in range(num_steps):
# Adaptation param and learning rate schedule as described in the paper
p = float(i) / num_steps
l = 2. / (1. + np.exp(-10. * p)) - 1
lr = 0.01 / (1. + 10 * p)**0.75
# Training step
if training_mode == 'dann':
#pdb.set_trace()
X0, y0 = gen_source_batch.next()
pdb.set_trace()
X1, y1 = gen_target_batch.next()
X = np.vstack([X0, X1])
y = np.vstack([y0, y1])
_, batch_loss, dloss, ploss, d_acc, p_acc = \
sess.run([dann_train_op, total_loss, domain_loss, pred_loss, domain_acc, label_acc],
feed_dict={model.X: X, model.y: y, model.domain: domain_labels,
model.train: True, model.l: l, learning_rate: lr})
if verbose and i % 100 == 0:
print 'loss: %f d_acc: %f p_acc: %f p: %f l: %f lr: %f' % \
(batch_loss, d_acc, p_acc, p, l, lr)
elif training_mode == 'source':
X, y = gen_source_only_batch.next()
print 'label_size', y.shape
pdb.set_trace()
_, batch_loss = sess.run([regular_train_op, pred_loss],
feed_dict={model.X: X, model.y: y, model.train: False,
model.l: l, learning_rate: lr})
elif training_mode == 'target':
X, y = gen_target_only_batch.next()
_, batch_loss = sess.run([regular_train_op, pred_loss],
feed_dict={model.X: X, model.y: y, model.train: False,
model.l: l, learning_rate: lr})
# Compute final evaluation on test data
source_acc = sess.run(label_acc,
feed_dict={model.X: mnist_test, model.y: mnist_test_labels,
model.train: False})
target_acc = sess.run(label_acc,
feed_dict={model.X: mnistm_test, model.y: mnist_test_labels,
model.train: False})
test_domain_acc = sess.run(domain_acc,
feed_dict={model.X: combined_test_imgs,
model.domain: combined_test_domain, model.l: 1.0})
test_emb = sess.run(model.feature, feed_dict={model.X: combined_test_imgs})
return source_acc, target_acc, test_domain_acc, test_emb
print '\nSource only training'
source_acc, target_acc, _, source_only_emb = train_and_evaluate('source', graph, model)
print 'Source (MNIST) accuracy:', source_acc
print 'Target (MNIST-M) accuracy:', target_acc
pdb.set_trace()
print '\nDomain adaptation training'
source_acc, target_acc, d_acc, dann_emb = train_and_evaluate('dann', graph, model)
print 'Source (MNIST) accuracy:', source_acc
print 'Target (MNIST-M) accuracy:', target_acc
print 'Domain accuracy:', d_acc
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000)
source_only_tsne = tsne.fit_transform(source_only_emb)
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000)
dann_tsne = tsne.fit_transform(dann_emb)
plot_embedding(source_only_tsne, combined_test_labels.argmax(1), combined_test_domain.argmax(1), 'Source only')
plot_embedding(dann_tsne, combined_test_labels.argmax(1), combined_test_domain.argmax(1), 'Domain Adaptation')
The error i am getting is:
Source train data size (5119, 20, 20, 3)
Source test data size (5119, 20, 20, 3)
Source train data size (9000, 20, 20, 3)
Source test data size (9000, 20, 20, 3)
Source test data size (9000, 20, 20, 1)
logit_size (?, 20)
logit_size (?, 20)
logit_size (?, 20)
Source only training
label_size (12, 20)
/usr/local/lib/python3.4/dist-packages/tensorflow/examples/tutorials/tf-dann-master/tf-DANN-LID-code_test.py(235)train_and_evaluate()
-> _, batch_loss = sess.run([regular_train_op, pred_loss],
(Pdb) c
Traceback (most recent call last):
File "tf-DANN-LID-code_test.py", line 265, in
print 'Source (MNIST) accuracy:', source_acc
File "tf-DANN-LID-code_test.py", line 235, in train_and_evaluate
feed_dict={model.X: X, model.y: y, model.train: False,
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 710, in run
run_metadata_ptr)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 908, in _run
feed_dict_string, options, run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 958, in _do_run
target_list, options, run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 978, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors.InvalidArgumentError: logits and labels must be same size: logits_size=[6,20] labels_size=[12,20]
[[Node: label_predictor/SoftmaxCrossEntropyWithLogits = SoftmaxCrossEntropyWithLogits[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](label_predictor/add_2, label_predictor/cond_1/Merge)]]
Caused by op u'label_predictor/SoftmaxCrossEntropyWithLogits', defined at:
File "tf-DANN-LID-code_test.py", line 160, in
File "tf-DANN-LID-code_test.py", line 80, in init
self._build_model()
File "tf-DANN-LID-code_test.py", line 137, in _build_model
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn_ops.py", line 491, in softmax_cross_entropy_with_logits
precise_logits, labels, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 1427, in _softmax_cross_entropy_with_logits
features=features, labels=labels, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2317, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1239, in init
self._traceback = _extract_stack()
The label size is (12,20) but logit size is (?,20).. How to fix this issue? Many Thanks
Regards
Saad