import string
import os.path
import sys
import time
import shutil
import ctools
import numpy as np

import spirouTHORCA
import spirouVISU
import hadmrMATH
import spirouFITS
import spirouBIAS
import spirouEXTOR
import spirouCDB
import hadrgdCONFIG
from numpy.oldnumeric import *
from extract2 import *
import matplotlib.pyplot as plt
import spirouRV

import fitpoly

s=string
FITS=spirouFITS
VISU=spirouVISU

tmp_file="%s"%int(time.time())

execfile(os.getenv('PYTHONSTARTUP'))
execfile(os.getenv('PYTHONSTARTUP2'))

###########################

Read Image FITS

nx=NAXIS1 (nb columns)

ny=NAXIS2 (nb rows)

data[ny,nx]

###########################

fitsfilename=os.path.join(dir_data_reduc,arg_night_name,arg_file_names[0])
dirfits=os.path.join(dir_data_reduc,arg_night_name)

if os.path.exists(dirfits)==0:
WLOG('error',log_opt,'Directory :'+dirfits+' does not exist')
exit(1)

if os.path.exists(fitsfilename)==0:
WLOG('error',log_opt,'File: '+fitsfilename+' does not exist')
exit(1)

if string.find(arg_file_names[0],'hc')<0:
WLOG('error',log_opt,'Wrong type of image for cal_HC, should be hc')
exit(1)

if string.find(arg_file_names[0],'AB')>0:
fiber='AB'
elif string.find(arg_file_names[0],'C')>0:
fiber='C'
else:
WLOG('error',log_opt,'Fiber name not recognized, should be AB or C')
exit(1)

delete matrices to free memory

if locals().has_key('data'): del data
if locals().has_key('data2'): del data2

WLOG('',log_opt,'Reading File: '+fitsfilename)
data,nx,nbo=spirouFITS.read_data_raw(fitsfilename)

if len(arg_file_names)>1:
for i in range(len(arg_file_names)-1):
fitsfilename=os.path.join(dir_data_reduc,arg_night_name,arg_file_names[i+1])
WLOG('',log_opt,'Reading Image '+fitsfilename+ ' and added')
data2,nx2,nbo2=spirouFITS.read_data_raw(fitsfilename)
data=add(data,data2)

WLOG('',log_opt,'Image '+repr(nx)+'x'+repr(nbo)+' loaded')

ccdgain=spirouFITS.read_key(fitsfilename,'GAIN')
ccdsigdet=spirouFITS.read_key(fitsfilename,'RDNOISE')
exptime=spirouFITS.read_key(fitsfilename,'EXPTIME')
bjdref=spirouFITS.read_key(fitsfilename,'ACQTIME')

kw_CCD_SIGDET[1]=ccdsigdet
kw_CCD_CONAD[1]=ccdgain

#########################

get caliDB

#########################

cal_db=spirouCDB.get_master(ic_calib_db_master_file,spirouCDB.filename2tunix2(fitsfilename),log_opt)
if cal_db.has_key('NONE'):
WLOG('error',log_opt,'Missing CalibDB can not proceed')
exit(0)
spirouCDB.cp_db_files(dir_calib_db,os.path.join(dir_data_reduc,arg_night_name),cal_db,log_opt)

Flat correction

if cal_db.has_key('FLAT_'+fiber):
WLOG('',log_opt+fiber,'Doing Flat Correction')
flat_file=os.path.join(dir_data_reduc,arg_night_name,cal_db['FLAT_'+fiber][1])
flat,nx,ny=spirouFITS.read_data_raw(flat_file)
else:
flat=ones(shape(data),'d')

flat=where(flat==0.,1.,flat)

#data=data/flat

#################################

define wave filename

#################################

wave_fitsfilename={
'AB': os.path.join(dir_data_reduc,arg_night_name,s.split(arg_night_name,'/')[1]+'_'+s.replace(arg_file_names[0],'_e2ds_AB.fits','wave_AB.fits')),
'C': os.path.join(dir_data_reduc,arg_night_name,s.split(arg_night_name,'/')[1]+''+s.replace(arg_file_names[0],'_e2ds_C.fits','_wave_C.fits'))}

#################################

start ll solution

#################################

Identifies the lamp

if string.find(arg_file_names[0],'hcone')>=0:
lamp='UNe'
elif string.find(arg_file_names[0],'hctwo')>=0:
lamp='TH'
else:
WLOG('error',log_opt,'Lamp type cannot be identified')
exit(1)

Identifies the lamp - once it's properly coded into the header

#lamp=FITS.read_key(fitsfilename,'INSLAMP')

th_LINES_={}
FINAL_ll_param_={}
FINAL_x_param_={}
FINAL_x_details_={}
FINAL_ll_details_={}
FINAL_iter_={}
ll_scale_={}
ll_out_={}
param_ll_out_={}
#param_x_out_={}
T_order_={}

#################################

redefine parameters

#################################

if lamp=='UNe':

ic_ll_TH_line_file='/home/spirou/mhobson/wavelength/catalogue_UNe_firstguess.dat' #UNe ref line list

cat='trimcat'

ic_ll_TH_line_file='/home/spirou/mhobson/wavelength/catalogue_UNe.dat'

ic_ll_TH_line_file=dir_drs_config+'catalogue_UNe.dat'
    cat='fullcat'

elif lamp=='TH':

ic_ll_TH_line_file='/home/spirou/mhobson/wavelength/catalogue_ThAr.dat' #ThAr ref line list

ic_ll_TH_line_file=dir_drs_config+'catalogue_ThAr.dat'
    cat='thcat'
#ic_ll_TH_line_file='/home/spirou/mhobson/wavelength/catalogue_ThAr_firstguess.dat'  #ThAr ref line list
#cat='thtrimcat'

ic_ll_degr_fit=4 #default is 5

ic_ll_sp_min=900
ic_ll_sp_max=2400
ic_max_ampl_lines=2.e+8 #2.e+5 # max ampl th lines
ic_max_errw_onfit=1 # 1
#ic_max_sigll_cal_lines=0.1 # 5.2
ic_Resol= 55000 #50000 #60000
ic_ll_free_span=3 #3 #2.6
noise=30 #16.8

n_ord_final=24 #order to which the solution is calculated
ic_T_order_start=66 #echelle of first extracted order

ic_ll_smooth=0

plt.ion()

fib_typ=[fiber]

for fiber in fib_typ:

#################################

wavelength solution

#################################

WLOG('info',log_opt+fiber,'Processing Wavelength Calibration for Fiber '+fiber)

first guess solution (consistency check)

T_order=-arange(0,n_ord_final,1)+ic_T_order_start*1.0

try:
cal_file=os.path.join(dir_data_reduc,arg_night_name,cal_db['WAVE_'+fiber][1]) ## To be used as soon as CalibDB has a good wavelentgh

cal_file='/home/spirou/mhobson/wavelength/testexptime/AT4-10:2017-08-15_11-51-34:hcone_hcone02c406_blue_wave_new722_C.fits' ##original solution - to replace if necessary

except KeyError:

cal_file='/home/spirou/mhobson/wavelength/testexptime/AT4-10:2017-08-15_11-51-34:hcone_hcone02c406_blue_wave_new722_C.fits'

cal_file=dir_drs_config+'hcone_hcone02c406_blue_wave_new722_C.fits'

ll_init,param_ll_init=FITS.get_e2ds_ll(cal_file) #new function no longer requires kw
WLOG('',log_opt+fiber,'Reading initial wavelength solution in '+cal_file)

ll_init=ll_init[:n_ord_final] #fit is performed only on these orders

linefile=open(ic_ll_TH_line_file,'r') #wavelength file
ll_line=[]
Ampl_line=[]
for line_txt in linefile.readlines():
ll_line.append(float(s.split(line_txt)[0]))
Ampl_line.append(float(s.split(line_txt)[1]))
linefile.close()
WLOG('',log_opt+fiber,'List of '+"%s"%(len(ll_line))+' HC lines read in file '+ic_ll_TH_line_file)

#################################

find ll on spectrum

#################################

WLOG('',log_opt+fiber,'On fiber '+fiber+' trying to identify lines using guess solution:')

Fix the bug on some line fit spirouTHORCA.py:197: RuntimeWarning: divide by zero encountered in double_scalars

th_LINES=spirouTHORCA.find_lines(ll_init,data[:n_ord_final],ic_ll_sp_min,ic_ll_sp_max,ic_Resol,
ic_ll_free_span,ll_line,Ampl_line,noise,log_opt+fiber,T_order)

#########################################################

detect bad fit filtering and saturated lines

#########################################################

WLOG('',log_opt+fiber,'On fiber '+fiber+' cleaning list of identified lines')

for i in (arange(len(th_LINES))):
bad_fit=sum(less_equal(th_LINES[i][:,7],ic_max_errw_onfit))
th_LINES[i][:,7]=choose(greater(th_LINES[i][:,7],ic_max_errw_onfit),
(zeros(len(th_LINES[i][:,7])),th_LINES[i][:,7]))
sigll=th_LINES[i][:,1]/th_LINES[i][:,0]*300000
th_LINES[i][:,7]=choose(less(sigll,ic_max_sigll_cal_lines),
(zeros(len(th_LINES[i][:,7])),th_LINES[i][:,7]))
bad_sig=sum(greater_equal(sigll,ic_max_sigll_cal_lines))
th_LINES[i][:,7]=choose(less(th_LINES[i][:,2],ic_max_ampl_lines),
(zeros(len(th_LINES[i][:,7])),th_LINES[i][:,7]))
bad_ampl=sum(greater_equal(th_LINES[i][:,2],ic_max_ampl_lines))
if (bad_sig>0)|(bad_ampl>0|(bad_fit>0)):
WLOG('',log_opt+fiber,'In Order '+"%s"%(i)+' reject ('+"%s/%s/%s"%(bad_sig,bad_ampl,bad_fit)+
') lines [beyond (sig/ampl/err) limits]')

      th_LINES[i][:,7]=np.choose(~np.isinf(th_LINES[i][:,7]),(zeros(len(th_LINES[i][:,7])),th_LINES[i][:,7])) #removing infs

#########################

fit llsol

#########################

WLOG('',log_opt+fiber,'On fiber '+fiber+' fitting wavelength solution on identified lines:')

FINAL_xfit_moy,FINAL_xfit_var,FINAL_iter,FINAL_x_param,FINAL_x_details,ll_scale=
spirouTHORCA.fit_1Ds_ll_sol(ll_init,th_LINES,ic_ll_degr_fit,ic_errx_min,ic_wei_max
,ic_max_llfit_rms,log_opt+fiber,T_order,fiber)

param_x_out=reshape(FINAL_x_param,(len(FINAL_x_param),len(FINAL_x_param[0])))

if (hadmrMATH.isInf(FINAL_xfit_moy))|(hadmrMATH.isNaN(FINAL_xfit_moy)):
QC_failed='NaN or Inf occured!'

return x=f(l,ord) -> l=F(x,ord)

FINAL_llfit_moy,FINAL_llfit_var,FINAL_ll_param,FINAL_ll_details=
spirouTHORCA.invert_1Ds_ll_sol(ll_init,FINAL_x_param,FINAL_x_details,
add.reduce(FINAL_iter[:,2]),ic_ll_degr_fit,log_opt+fiber)

param_ll_out=reshape(FINAL_ll_param,(len(FINAL_ll_param),len(FINAL_ll_param[0])))
ll_out=spirouTHORCA.get_ll(len(data[:n_ord_final][0]),len(FINAL_ll_param),param_ll_out)
dll_out=spirouTHORCA.get_dll(len(data[:n_ord_final][0]),len(FINAL_ll_param),param_ll_out)
WLOG('info',log_opt+fiber,'On fiber '+fiber+' mean pixel scale at center: '+
"%.4f"%(sum(dll_out[:,len(ll_out[0])/2]/ll_out[:,len(ll_out[0])/2])/len(ll_out[:,len(ll_out[0])/2])*300000)+'[km/s/pixel]')

####### Extrapolation of param_ll_out for all spectral orders - NOT USED TO BE REMOVED LATER

##zeroth coefficients

coe0=np.zeros(3,'d') #matrix for the coefficients of the fits

test0=np.zeros(len(param_ll_out[:,0]),'d')

fitpoly.fitpoly(np.arange(24)*1.0,param_ll_out[:,0],coe0,test0)

##first coefficients

coe1=np.zeros(3,'d') #matrix for the coefficients of the fits

test1=np.zeros(len(param_ll_out[:,1]),'d')

fitpoly.fitpoly(np.arange(24)*1.0,param_ll_out[:,1],coe1,test1)

##second coefficients

coe2=np.zeros(2,'d') #matrix for the coefficients of the fits

test2=np.zeros(len(param_ll_out[:,2]),'d')

fitpoly.fitpoly(np.arange(24)*1.0,param_ll_out[:,2],coe2,test2)

##third coefficients

coe3=np.zeros(2,'d') #matrix for the coefficients of the fits

test3=np.zeros(len(param_ll_out[:,3]),'d')

fitpoly.fitpoly(np.arange(24)*1.0,param_ll_out[:,3],coe3,test3)

##fourth coefficients

coe4=np.zeros(2,'d') #matrix for the coefficients of the fits

test4=np.zeros(len(param_ll_out[:,4]),'d')

fitpoly.fitpoly(np.arange(24)*1.0,param_ll_out[:,4],coe4,test4)

extrapolating to the redder orders

param_ll_out_extrap=np.zeros((36,ic_ll_degr_fit+1),'d')

for i in range(n_ord_final):

param_ll_out_extrap[i]=param_ll_out[i]

for i in range(n_ord_final,36):

for k in range(len(coe0)):

param_ll_out_extrap[i,0]+=coe0[k]*i**k

for k in range(len(coe1)):

param_ll_out_extrap[i,1]+=coe1[k]*i**k

for k in range(len(coe2)):

param_ll_out_extrap[i,2]+=coe2[k]*i**k

for k in range(len(coe3)):

param_ll_out_extrap[i,3]+=coe3[k]*i**k

for k in range(len(coe4)):

param_ll_out_extrap[i,4]+=coe4[k]*i**k

ll_out_full=spirouTHORCA.get_ll(len(data[0]),len(param_ll_out_extrap),param_ll_out_extrap)

dll_out_full=spirouTHORCA.get_dll(len(data[0]),len(param_ll_out_extrap),param_ll_out_extrap)

WLOG('info',log_opt+fiber,'On fiber '+fiber+' mean pixel scale at center: '+\

"%.4f"%(sum(dll_out_full[:,len(ll_out_full[0])/2]/ll_out_full[:,len(ll_out_full[0])/2])/len(ll_out_full[:,len(ll_out_full[0])/2])*300000)+'[km/s/pixel]')

#########################################

extrapolate Littrow solution

#########################################

th_LINES_[fiber]=th_LINES
FINAL_ll_param_[fiber]=FINAL_ll_param
FINAL_x_param_[fiber]=FINAL_x_param
FINAL_x_details_[fiber]=FINAL_x_details
FINAL_ll_details_[fiber]=FINAL_ll_details
FINAL_iter_[fiber]=FINAL_iter
ll_out_[fiber]=ll_out
ll_scale_[fiber]=ll_scale
param_ll_out_[fiber]=param_ll_out

param_x_out_[fiber]=param_x_out

T_order_[fiber]=T_order
N_blue_ord=n_ord_final
ic_order_init=0
ic_Littrow_fit_deg=5

sig_Littrow_ext=[]
moy_Littrow_ext=[]
mindev_Littrow_ext=[]
maxdev_Littrow_ext=[]
save_Littrow_param=[]

x_Littrow_cut_ext=[250,500,750,1000,1250,1500,1750] #points at which to evaluate the Littrow fit on each order

wgt_Littrow=T_order_[fiber]*0+1
if locals().has_key('ic_wgt_Littrow'):
for iorder in ic_wgt_Littrow[fiber]:
wgt_Littrow[iorder]=0.

for x in range(len(x_Littrow_cut_ext)): #do Littrow fit
Q=spirouTHORCA.view_Littrow(T_order_,ll_out_,fiber,wgt_Littrow,x_Littrow_cut_ext[x],ic_Littrow_fit_deg,N_blue_ord,ic_order_init,x>0,0,ic_debug)
moy_Littrow_ext.append(Q[0])
sig_Littrow_ext.append(Q[1])
mindev_Littrow_ext.append(Q[2])
maxdev_Littrow_ext.append(Q[3])
save_Littrow_param.append(Q[6])

Littrow_extrap = np.zeros((36,len(x_Littrow_cut_ext)),'d') #stores evaluated Littrow fits for the x_ext values on each order
Littrow_extrap_sol = np.zeros((36,len(data[0])),'d') #stores ll(x) for each order
param_Littrow_extrap=np.zeros((36,5),'d') #stores Littrow params for each order

for i in range(36): #evaluate the Littrow fit on the x_..._ext values for each order
for j in range(len(x_Littrow_cut_ext)):
aux=[]
for k in range(len(save_Littrow_param[j])):
aux.append(save_Littrow_param[j][k]*(1./(ic_T_order_start-i))**(k))
Littrow_extrap[i,j]=sum(aux)*ll_out[ic_order_init][x_Littrow_cut_ext[j]]

param=np.zeros(5,'d')
auxx=np.zeros(len(x_Littrow_cut_ext),'d')
fitpoly.fitpoly(x_Littrow_cut_ext,Littrow_extrap[i],param,auxx)	#fit a polynomial to the 7 evaluated Littrow ll(x) on each order
param_Littrow_extrap[i]=param

for jj in range(len(data[0])):	#evaluate the polyinomial in each x in (0,2035) for each order
  aux2=[]
  for kk in range(len(param)):
    aux2.append(param[kk]*jj**kk)
  Littrow_extrap_sol[i,jj]=sum(aux2)

plt.figure() #plot the points and function for each order
for i in range(36):
plt.plot(x_Littrow_cut_ext,Littrow_extrap[i],'o')
plt.plot(np.arange(len(data[0])),Littrow_extrap_sol[i])

#########################################

repeat the line search loop

#########################################

ic_ll_free_span=2.6 #3 #reducing slightly as it's the second loop
n_ord_final_2=24 # can't go above 34 - above 24 is more distorted, less consistent for passing QC
T_order=-arange(0,n_ord_final_2,1)+ic_T_order_start*1.0

preselect lines to use only those kept in the first line search for orders 0-24

ll_line_2=[]
Ampl_line_2=[]
for i in range(n_ord_final):
for j in range(len(FINAL_x_details[i][0])):
ll_line_2.append(FINAL_x_details[i][0][j])
Ampl_line_2.append(FINAL_x_details[i][3][j])

maxll=max(ll_line_2)

add the full cat for the redder orders

for i in range(len(ll_line)):
if ll_line[i]>maxll:
ll_line_2.append(ll_line[i])
Ampl_line_2.append(Ampl_line[i])

find ll on spectrum

WLOG('',log_opt+fiber,'On fiber '+fiber+' trying to identify lines using guess solution (second pass):')

th_LINES_2=spirouTHORCA.find_lines(Littrow_extrap_sol[:n_ord_final_2],data[:n_ord_final_2],ic_ll_sp_min,ic_ll_sp_max,ic_Resol,
ic_ll_free_span,ll_line_2,Ampl_line_2,noise,log_opt+fiber,T_order)

detect bad fit filtering and saturated lines

WLOG('',log_opt+fiber,'On fiber '+fiber+' cleaning list of identified lines (second pass)')

for i in (arange(len(th_LINES_2))):
bad_fit=sum(less_equal(th_LINES_2[i][:,7],ic_max_errw_onfit))
th_LINES_2[i][:,7]=choose(greater(th_LINES_2[i][:,7],ic_max_errw_onfit),
(zeros(len(th_LINES_2[i][:,7])),th_LINES_2[i][:,7]))
sigll=th_LINES_2[i][:,1]/th_LINES_2[i][:,0]*300000
th_LINES_2[i][:,7]=choose(less(sigll,ic_max_sigll_cal_lines),
(zeros(len(th_LINES_2[i][:,7])),th_LINES_2[i][:,7]))
bad_sig=sum(greater_equal(sigll,ic_max_sigll_cal_lines))
th_LINES_2[i][:,7]=choose(less(th_LINES_2[i][:,2],ic_max_ampl_lines),
(zeros(len(th_LINES_2[i][:,7])),th_LINES_2[i][:,7]))
bad_ampl=sum(greater_equal(th_LINES_2[i][:,2],ic_max_ampl_lines))
if (bad_sig>0)|(bad_ampl>0|(bad_fit>0)):
WLOG('',log_opt+fiber,'In Order '+"%s"%(i)+' reject ('+"%s/%s/%s"%(bad_sig,bad_ampl,bad_fit)+
') lines [beyond (sig/ampl/err) limits]')

      th_LINES_2[i][:,7]=np.choose(~np.isinf(th_LINES_2[i][:,7]),(zeros(len(th_LINES_2[i][:,7])),th_LINES_2[i][:,7])) #removing infs

WLOG('',log_opt+fiber,'On fiber '+fiber+' fitting wavelength solution on identified lines (second pass):')

fit llsol

FINAL_xfit_moy,FINAL_xfit_var,FINAL_iter,FINAL_x_param,FINAL_x_details,ll_scale=
spirouTHORCA.fit_1Ds_ll_sol(Littrow_extrap_sol[:n_ord_final_2],th_LINES_2,ic_ll_degr_fit,ic_errx_min,ic_wei_max
,ic_max_llfit_rms,log_opt+fiber,T_order,fiber)

param_x_out=reshape(FINAL_x_param,(len(FINAL_x_param),len(FINAL_x_param[0])))

if (hadmrMATH.isInf(FINAL_xfit_moy))|(hadmrMATH.isNaN(FINAL_xfit_moy)):
QC_failed='NaN or Inf occured!'

return x=f(l,ord) -> l=F(x,ord)

FINAL_llfit_moy,FINAL_llfit_var,FINAL_ll_param,FINAL_ll_details=
spirouTHORCA.invert_1Ds_ll_sol(Littrow_extrap_sol[:n_ord_final_2],FINAL_x_param,FINAL_x_details,
add.reduce(FINAL_iter[:,2]),ic_ll_degr_fit,log_opt+fiber)

param_ll_out_2=reshape(FINAL_ll_param,(len(FINAL_ll_param),len(FINAL_ll_param[0])))
ll_out_2=spirouTHORCA.get_ll(len(data[:n_ord_final_2][0]),len(FINAL_ll_param),param_ll_out_2)
dll_out_2=spirouTHORCA.get_dll(len(data[:n_ord_final_2][0]),len(FINAL_ll_param),param_ll_out_2)

WLOG('info',log_opt+fiber,'On fiber '+fiber+' mean pixel scale at center: '+
"%.4f"%(sum(dll_out_2[:,len(ll_out_2[0])/2]/ll_out_2[:,len(ll_out_2[0])/2])/len(ll_out_2[:,len(ll_out_2[0])/2])*300000)+'[km/s/pixel]')

#########################

Littrow test

#########################

ic_x_Littrow_cut=[500,1000,1500]

th_LINES_[fiber]=th_LINES
FINAL_ll_param_[fiber]=FINAL_ll_param
FINAL_x_param_[fiber]=FINAL_x_param
FINAL_x_details_[fiber]=FINAL_x_details
FINAL_ll_details_[fiber]=FINAL_ll_details
FINAL_iter_[fiber]=FINAL_iter
ll_out_[fiber]=ll_out_2
ll_scale_[fiber]=ll_scale
param_ll_out_[fiber]=param_ll_out_2

param_x_out_[fiber]=param_x_out

T_order_[fiber]=T_order
N_blue_ord=n_ord_final_2
ic_order_init=0
ic_Littrow_fit_deg=5

sig_Littrow=[]
moy_Littrow=[]
mindev_Littrow=[]
maxdev_Littrow=[]
x_Littrow_cut=ic_x_Littrow_cut
wgt_Littrow=T_order_[fiber]*0+1
if locals().has_key('ic_wgt_Littrow'):
for iorder in ic_wgt_Littrow[fiber]:
wgt_Littrow[iorder]=0.

plt.figure()
plt.clf()
for x in range(len(x_Littrow_cut)):
Q=spirouTHORCA.view_Littrow(T_order_,ll_out_,fiber,wgt_Littrow,x_Littrow_cut[x],ic_Littrow_fit_deg,N_blue_ord,ic_order_init,x>0,0,ic_debug)
moy_Littrow.append(Q[0])
sig_Littrow.append(Q[1])
mindev_Littrow.append(Q[2])
maxdev_Littrow.append(Q[3])
WLOG('info',log_opt+fiber,'Littrow check at X='+"%s"%(x_Littrow_cut[x])+' mean:'+"%6.3f"%(Q[0]*1000)+'[m/s] rms:'+
"%6.2f"%(Q[1]*1000)+' [m/s] min/max:'+"%6.2f/%6.2f [m/s]"%(Q[2]*1000,Q[3]*1000)+
" (frac: %4.1f/%4.1f)"%(Q[2]/Q[1],Q[3]/Q[1]))
VISU.megaplot(Q[4],Q[5],xtitle='#order',ytitle='Diff/Littrow [km/s]',
title='Wavelength Solution Littrow Check fiber '+fiber,
xmin=int(Q[4][0]+1),xmax=int(Q[4][-1]-1),ymin=-1,ymax=1,overplot=x>0)

#########################################

join 0-24 and 25-36 solutions

#########################################

ll_out_3=np.vstack((ll_out_2,Littrow_extrap_sol[n_ord_final_2:]))
param_ll_out_3=np.vstack((param_ll_out_2,param_Littrow_extrap[n_ord_final_2:]))

#########################################

archive result in e2ds spectra

#########################################

WLOG('',log_opt,'Write wavelength Solution of Fiber '+fiber+' in '+os.path.split(fitsfilename)[1])

spirouFITS.put_e2ds_ll_param(fitsfilename,param_ll_out_3)

Storage in wave "spectra"

WLOG('',log_opt,'Saving wavelength array of Fiber '+fiber+' in '+os.path.split(wave_fitsfilename[fiber])[1])
spirouFITS.write_data(wave_fitsfilename[fiber], ll_out_3)

Copy keywords of the e2ds frame

spirouFITS.copy_key(fitsfilename,wave_fitsfilename[fiber])

#################################

print tbl result

#################################

res_tbl_file=os.path.join(dir_data_reduc,arg_night_name,'cal_HC_result.tbl')
try:
file=open(res_tbl_file)
except IOError:
file=open(res_tbl_file,'w')
file.write('night_name'+ '\t' +'file_name'+ '\t' +'fiber'+ '\t' +'mean'+
'\t' +'rms'+ '\t' +'N_lines'+ '\t' +'err'+
'\t' +'rms_L0'+ '\t'+'rms_L1'+ '\t'+'rms_L2'+'\t'+'drift'+'\t'+'drift2'+'\t'+'Rflux'+'\t'+'Ccosmic'+
'\t' +'error_spe'+'\n')
file.write('----------'+ '\t' +'---------'+ '\t' +'-----'+ '\t' +'---'+
'\t' +'---'+ '\t' +'-------'+ '\t' +'---'+
'\t' +'------'+ '\t'+'------'+ '\t'+'------'+'\t'+'-----'+'\t'+'------'+'\t'+'-----'+'\t'+'-------'+
'\t' +'---------'+'\n')
file.close()
os.chmod(res_tbl_file,0664)
try:
file=open(res_tbl_file,'a')
except IOError:
WLOG('error',log_opt+fiber,'I/O error on file: '+res_tbl_file)
exit(0)
file.write(arg_night_name+'\t'+arg_file_names[0]+'\t'+fiber+'\t'+
"%7.4f\t"%(FINAL_xfit_moy*1000)+
"%6.2f"%(sqrt(FINAL_xfit_var)*1000)+'\t'+
"%i"%add.reduce(FINAL_iter[:,2])+'\t'+
"%6.3f"%(sqrt(FINAL_xfit_var/add.reduce(FINAL_iter[:,2]))*1000)+'\t'+
"%6.2f"%(sig_Littrow[0]*1000)+'\t'+
"%6.2f"%(sig_Littrow[1]*1000)+'\t'+
"%6.2f\n"%(sig_Littrow[2]*1000))
file.close()
WLOG('',log_opt+fiber,'Global result summary saved in '+res_tbl_file)

#################################

PRINT th_line tbl file

#################################

res_tbl_line_file=os.path.join(dir_data_reduc,arg_night_name, s.replace(arg_file_names[0],'.fits','hc_lines'+fiber+'.tbl'))
file=open(res_tbl_line_file,'w')
file.write('order\tll\tdv\tw\txi\txo\tdvdx\n')
file.write('-----\t--\t--\t-\t--\t--\t----\n')
for i in range(len(FINAL_x_details)):
for j in range(len(FINAL_x_details[i][0])):
file.write("%2i"%i+'\t'+"%12.4f"%FINAL_x_details[i][0][j]+'\t'+"%13.5f"%FINAL_ll_details[i][0][j]+'\t'
+"%12.4f"%FINAL_x_details[i][3][j]+'\t'+"%12.4f"%FINAL_x_details[i][1][j]+'\t'+"%12.4f"%FINAL_x_details[i][2][j]+'\t'+"%8.4f"%ll_scale[i][j]+'\n')
file.close()
WLOG('',log_opt+fiber,'List of lines used saved in '+res_tbl_line_file)

#################

QC params

#################

qc_rms_littrow_max=[0.300,0.300,0.300] # max Littrow deviation for ll sol at X=700,1200,1700
qc_dev_littrow_max=[0.900,0.900,0.900] # max Littrow

if not(locals().has_key('QC_failed')):
for x in range(len(x_Littrow_cut)):
if (sig_Littrow[x] >qc_rms_littrow_max[x])|
(max(abs(maxdev_Littrow[x]),abs(mindev_Littrow[x])) >qc_dev_littrow_max[x]):
QC_failed='Littrow test x='+"%s"%(x_Littrow_cut[x])
WLOG('error',log_opt+fiber,'On fiber '+fiber+ ' QC failed for: '+QC_failed)

if locals().has_key('QC_failed'): 
   kw_drs_QC[1]='FAILED'
else:
   WLOG('info',log_opt+fiber,'On fiber '+fiber+' QUALITY CONTROL SUCCESSFUL  - Well Done -')

#########################

Add QC kw

#########################

spirouFITS.update_key_2(fitsfilename,kw_drs_QC)
spirouFITS.update_key_2(wave_fitsfilename[fiber],kw_drs_QC)

#########################

compute CCF

#########################

wave={}
param_ll={}
cor_fitsfilename={
'AB': os.path.join(dir_data_reduc, arg_night_name, string.replace(arg_file_names[0],
'.fits','ccf'+'HC'+'_AB.fits')),
'C': os.path.join(dir_data_reduc, arg_night_name, string.replace(arg_file_names[0],
'.fits','ccf'+'HC'+'_C.fits'))}

res_tbl_file={
'C': string.replace(cor_fitsfilename['C'],'.fits','.tbl'),
'AB': string.replace(cor_fitsfilename['AB'],'.fits','.tbl')}

for fiber in fib_typ:
if lamp=='UNe':

ic_hc_mask='/home/spirou/mhobson/wavelength/test_mask_UNe_firstguess_R50000.mas'

 ic_hc_mask=dir_drs_config+'test_mask_UNe_firstguess_R50000.mas'

elif lamp=='TH':

ic_hc_mask='/home/spirou/mhobson/wavelength/test_mask_TH_R50000.mas'

 ic_hc_mask=dir_drs_config+'test_mask_TH_R50000.mas'

wave[fiber],param_ll[fiber]=spirouFITS.get_e2ds_ll(fitsfilename)
RV_template=ic_hc_mask
ll_mask_D,ll_mask_ctr,w_mask=spirouRV.get_mask(RV_template,1,0,log_opt)

ic_thorium_CCF_half_width=10
ic_thorium_CCF_step=0.1

RV_CCF=arange(-ic_thorium_CCF_half_width,ic_thorium_CCF_half_width+ic_thorium_CCF_step,ic_thorium_CCF_step)
CCF,CCF_max,pix_passed_all,tot_line,ll_range_all,CCF_noise=
spirouRV.coravelation(wave[fiber],data,param_ll[fiber],data*0.+1.,
RV_CCF,ll_mask_D,ll_mask_ctr,w_mask,0,0,noise,res_tbl_file[fiber],1,0)

average_CCF=sum(CCF)
normalized_CCF=average_CCF/max(average_CCF)
CCF_res,CCF_fit=spirouRV.fit_CCF(RV_CCF,normalized_CCF,1)

redo centered

RV_CCF=arange(CCF_res[1]-ic_thorium_CCF_half_width,CCF_res[1]+ic_thorium_CCF_half_width+ic_thorium_CCF_step,ic_thorium_CCF_step)
CCF,CCF_max,pix_passed_all,tot_line,ll_range_all,CCF_noise=
spirouRV.coravelation(wave[fiber],data,param_ll[fiber],data*0.+1.,
RV_CCF,ll_mask_D,ll_mask_ctr,w_mask,0,0,noise,res_tbl_file[fiber],1,0)

average_CCF=sum(CCF)
normalized_CCF=average_CCF/max(average_CCF)
CCF_res,CCF_fit=spirouRV.fit_CCF(RV_CCF,normalized_CCF,1)

CCF_res[0]=CCF_res[0]/(1.+CCF_res[3])

WLOG('info',log_opt,fiber+': Correlation: '+\

' C='+"%.1f"%(abs(100*CCF_res[0]))+'[%]'+\

' RV='+"%.1f"%(CCF_res[1]*1000)+'[m/s]'+\

' FWHM='+"%.3f"%(CCF_res[2]*2.3548)+'[km/s]'+\

' maxcpp='+"%.1f"%(sum(CCF_max)/sum(pix_passed_all)))

summary='calibrated CCF at RV= %.3f km/s'%(CCF_res[1])

if os.environ.has_key('DRS_PLOT') & (not locals().has_key('QC_failed')):

if os.environ['DRS_PLOT']=='trigger':

hadgtVISU.printfic('/tmp/plotA.'+tmp_file,RV_CCF,normalized_CCF)

WLOG('graph',log_opt,"hadgtVISU.megaplotfic('/tmp/plotA."+tmp_file+"',ymax=1.1,ymin=0,xmin="+'%s'%int(RV_CCF[0]-1)+",xmax="+'%s'%int(RV_CCF[-1]+1)+")")

hadgtVISU.printfic('/tmp/plotB.'+tmp_file,RV_CCF,CCF_fit)

WLOG('graph',log_opt,"hadgtVISU.megaplotfic('/tmp/plotB."+tmp_file+"',ymax=1.1,ymin=0,xmin="+'%s'%int(RV_CCF[0]-1)+",xmax="+'%s'%int(RV_CCF[-1]+1)+",overplot=1,xtitle='km/s',ytitle='CCF',title='Average CCF normalized of thorium\\n "+summary+"')")

else:

hadgtVISU.megaplot(RV_CCF,normalized_CCF,ymax=1.1,ymin=0,xmin=int(RV_CCF[0]-1),xmax=int(RV_CCF[-1]+1))

hadgtVISU.megaplot(RV_CCF,CCF_fit,ymax=1.1,ymin=0,xmin=int(RV_CCF[0]-1),xmax=int(RV_CCF[-1]+1),overplot=1,xtitle='km/s',ytitle='CCF',title='Average CCF normalized of thorium\n'+summary)

#########################

archive in fits

#########################

_kw_ccf={'CCF_MXCP': ['CCFMACPP', 'max count/pixel of CCF (e-)'], 'CCF_MASK': ['CCFMASK', 'Mask filename'], 'CCF_RVC': ['CCFRVC', 'Baryc RV (drift corrected) (km/s) '], 'CCF_CONT': ['CCFCONTR', 'Contrast of CCF (%)'], 'CCF_FWHM': ['CCFFWHM', 'FWHM of CCF (km/s)'], 'CCF_RV': ['CCFRV', 'Baryc RV (no drift correction) (km/s)'], 'CCF_LINE': ['CCFLINES', 'nbr of lines used']}

WLOG('',log_opt,'Archiving CCF on file: '+cor_fitsfilename[fiber])

spirouFITS.Save_ccf(RV_CCF,CCF,sum(CCF),cor_fitsfilename[fiber],ic_hc_mask,\

sum(CCF_max)/sum(pix_passed_all),sum(tot_line),abs(100*CCF_res[0]),CCF_res[1],CCF_res[2]*2.3548,CCF_res[1],_kw_CCF)

copy e2ds file kw into CCF

spirouFITS.copy_key(fitsfilename,cor_fitsfilename[fiber])

#########################

update calibDB

#########################

e2dscopy_fitsfilename = os.path.split(fitsfilename)[0]+'/'+s.split(arg_night_name,'/')[1]+'_'+os.path.split(fitsfilename)[1]
shutil.copyfile(fitsfilename,e2dscopy_fitsfilename) #makes a properly-named copy of the e2ds file for the calibdb

for fiber in fib_typ:
if not(locals().has_key('QC_failed')):
spirouCDB.put_file(dir_calib_db,wave_fitsfilename[fiber],log_opt)
spirouCDB.update_master(ic_calib_db_master_file,arg_night_name,'WAVE_'+fiber,os.path.split(wave_fitsfilename[fiber])[1],log_opt+fiber)
spirouCDB.put_file(dir_calib_db,e2dscopy_fitsfilename,log_opt)
spirouCDB.update_master(ic_calib_db_master_file,arg_night_name,'HCREF_'+fiber,os.path.split(e2dscopy_fitsfilename)[1],log_opt+fiber)

CDB.put_file(dir_calib_db,cor_fitsfilename[fiber],log_opt)

CDB.update_master(ic_calib_db_master_file,arg_night_name,'THCCF_'+fiber,os.path.split(cor_fitsfilename[fiber])[1],log_opt+fiber)

CDB.put_file(dir_calib_db,res_tbl_file[fiber],log_opt)

CDB.update_master(ic_calib_db_master_file,arg_night_name,'THRV_'+fiber,os.path.split(res_tbl_file[fiber])[1],log_opt+fiber)

WLOG('info',log_opt,'Recipe '+process_running+' has been successfully completed')

####################################################
def view_iter(order,iter,fiber):
###################################################
wei=THORCA.fit_sol.all_details[order][iter][3]
x=THORCA.fit_sol.all_details[order][iter][0]
diff=(THORCA.fit_sol.all_details[order][iter][2]-THORCA.fit_sol.all_details[order][iter][1])
for i in range(len(x)):
print "%6.2f"%(x[i])+"%8.2f"%(diff[i])+"%8.2f"%(1/sqrt(wei[i]))+"%8.2f"%(diff[i]sqrt(wei[i]))
VISU.megaplot(x,diffsqrt(wei),typo='points',xmin=x[0],xmax=x[-1])
VISU.megaplot(x,diff/max(diff),overplot=1,typo='points')
VISU.megaplot(x,diff/max(diff),overplot=1,typo='points')

####################################################
def view_iter2(order,iter,fiber):
###################################################
wei=THORCA.fit_sol.all_details[order][iter][3]
x=THORCA.fit_sol.all_details[order][iter][0]
diff=(THORCA.fit_sol.all_details[order][iter][2]-THORCA.fit_sol.all_details[order][iter][1])
VISU.Points(x,diff,0,0,0,0,0,
'iter:'+"%s"%iter+' rms:'+"%5.3f"%sqrt(THORCA.fit_sol.all_iter[order][iter][1]))
L=ll_out_[fiber][order,:]
F=data[:n_ord_final][order,:]
VISU.Histo2(L,F,0,0,5000,0,0,'')