epoch = 0.004
epoch = 0.104
epoch = 0.204
epoch = 0.304
epoch = 0.404
epoch = 0.504
epoch = 0.604
epoch = 0.703

[pavels@jones-6 pavels]$ warning: fbcunn not found. Falling back to cunn
Using 1-th gpu
Loading /local/pavels/lstm_zaremba/data/ptb.train.txt, size of data = 598835
Loading /local/pavels/lstm_zaremba/data/ptb.valid.txt, size of data = 54291
Loading /local/pavels/lstm_zaremba/data/ptb.test.txt, size of data = 53432
Network parameters:
{
  max_grad_norm : 5
  seq_length : 20
  batch_size : 20
  lr : 1
  max_max_epoch : 13
  rnn_size : 200
  init_weight : 0.1
  decay : 2
  dropout : 0
  layers : 2
  vocab_size : 10000
  max_epoch : 4
}
Creating a RNN LSTM network.
Starting training.
epoch = 0.007, train perp. = 0.000, wps = 3693, dw:norm() = 4.650, lr = 1.000, since beginning = 0 mins.
THCudaCheck FAIL file=/local/pavels/torch/extra/cutorch/init.c line=230 error=77 : an illegal memory access was encountered
/local/pavels/torch/install/bin/luajit: cuda runtime error (77) : an illegal memory access was encountered at /local/pavels/torch/extra/cutorch/lib/THC/generic/THCStorage.c:147
^C

replicate(x, 20)
  1   7  12  18  23  29  35  28  43  46  29  51  56  58  61  66  36  75  78  28
  2   8  13  19  24  30  36  25  27  47  30  52  36  59  43  67  72  76  65  29
  3   9  14  20  25  31  37  40  44  25  49  53  37  60  62  68  73  36  79  82
  4  10  15  21  26  32  38  27  33  48  50  54  38  25  63  69  43  47  80  83
  5  11  16  22  27  33  39  41  45  27  42  55  43  36  64  70  74  43  81  84
[torch.DoubleTensor of size 5x20]

  reset_ds()
  for i = params.seq_length, 1, -1 do
    state.pos = state.pos - 1
    local x = state.data[state.pos]
    local y = state.data[state.pos + 1]
    local s = model.s[i - 1]
    local derr = transfer_data(torch.ones(1))
    local tmp = model.rnns[i]:backward({x, y, s},
                                       {derr, model.ds})[3]
    g_replace_table(model.ds, tmp)
    cutorch.synchronize()
  end

local function main()
   --torch.setnumthreads(10)
   --print('threads: ', torch.getnumthreads())
   g_make_deterministic(1)
   
   state_train = {data=ptb.traindataset(params.batch_size)}
   state_valid = {data=ptb.validdataset(params.batch_size)}
   state_test  = {data=ptb.testdataset(params.batch_size)}
   params.vocab_size = ptb.vocab_size()
   print('Network parameters')
   print(params)
      
   local states = {state_train, state_valid, state_test}
   for _, state in pairs(states) do
      reset_state(state)
   end

   setup()

   -- load saved model before train
   local saved_model
   local file = io.open(params.model_path, "rb")
   if file then
      file:close()
      saved_model = torch.load(params.model_path)
      print('load from previous saved model')
   end
   
   model = saved_model or model
   collectgarbage()

   state_train.pos = model.state_train_pos or 1
   params.lr = model.lr or params.lr
   local step = model.step or 0
   local epoch = model.epoch or 0
   local total_cases = model.total_cases or 0
   local tics = model.tics or 0
   local beginning_time = torch.tic() - tics
   local start_time = torch.tic() - tics
   print('Starting training')
   local words_per_step = params.seq_length * params.batch_size
   local epoch_size = torch.floor(state_train.data:size(1) / params.seq_length)
   local perps
   while epoch < params.max_max_epoch do
      local perp = fp(state_train)
      if perps == nil then
	 perps = torch.zeros(epoch_size):add(perp)
      end
      perps[step % epoch_size + 1] = perp
      step = step + 1
      bp(state_train)
      total_cases = total_cases + params.seq_length * params.batch_size
      epoch = step /epoch_size
      if step % torch.round(epoch_size / 10) == 10 then
	 local wps = torch.floor(total_cases / torch.toc(start_time))
	 local since_beginning = g_d(torch.toc(beginning_time)/60)
	 print('epoch = ' .. g_f3(epoch) ..
		  ', train perp. = ' .. g_f3(torch.exp(perps:mean()))..
		  ', wps = ' .. wps ..
		  ', dw:norm() = ' .. g_f3(model.norm_dw) ..
		  ', lr = ' .. g_f3(params.lr) ..
		  ', since begining = '..since_beginning..' mins')

	 -- save model to model_path file
	 model.step = step
	 model.epoch = epoch
	 model.total_cases = total_cases
	 model.tics = torch.tic() - beginning_time
	 model.state_train_pos = state_train.pos
	 model.lr = params.lr
	 --clear_state()
	 torch.save(params.model_path, model)
	 
      end
      if step % epoch_size == 0 then
	 run_valid()
	 if epoch > params.max_epoch then
	    params.lr = params.lr / params.decay
	 end
      end
      if step % 33 == 0 then
	 collectgarbage()
      end
   end
   run_test()
   print('training is over.')
end