Collection of scripts and small programs used by the EVN Support Scientists at JIVE during the regular data processing of EVN observations.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

267 lines
7.9 KiB

  1. #! /bin/zsh
  2. function post_process_eee() {
  3. # Three parameters are expected:
  4. # - The experiment name (case insensitive).
  5. # - The reference station to use in standardplots.
  6. # - The calibrators to use in standardplots.
  7. if [[ ! ( -n $1 && -n $2 && -n $3 ) ]];then
  8. echo "Three parameters are required:"
  9. echo " - experiment name (case insensitive)."
  10. echo " - Reference station to be used for the plots."
  11. echo " - Calibrators to be used for the plots."
  12. exit
  13. fi
  14. exp=$1
  15. exp=${(L)1}
  16. EXP=${(U)1}
  17. date=$(ssh jops@ccs grep ${EXP} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d ' ' -f 3)
  18. # In the case of eEVN with an experiment name different this method may not work
  19. if [[ ! -n $date ]];then
  20. date=$(ssh jops@ccs grep ${EXP} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d ' ' -f 4)
  21. fi
  22. # Sometimes it has a \n or empty spaces.
  23. date=${${${date}:s/"\\n"/""}:s/" "/""}
  24. echo "Processing experiment ${EXP}_${date}.\n"
  25. # Creating the experiment directory in /data0/marcote/EXP and moving to it
  26. if [ ! -d /data0/marcote/$EXP ]; then
  27. mkdir /data0/marcote/$EXP
  28. fi
  29. cd /data0/marcote/$EXP
  30. # Password for
  31. export pass=$(date | md5sum | cut -b 1-12)
  32. create_processing_log.py $EXP -p $pass
  33. # Create the lis file from ccs
  34. ssh jops@ccs "cd /ccs/expr/${EXP};/ccs/bin/make_lis -e ${EXP} -p prod -s ${exp}.lis"
  35. scp jops@ccs:/ccs/expr/${EXP}/${exp}.vix ./${exp}.vix
  36. scp jops@ccs:/ccs/expr/${EXP}/${exp}.lis ./${exp}.lis
  37. scp jops@jop83:piletters/${exp}.piletter .
  38. scp jops@jop83:piletters/${exp}.expsum .
  39. ln -s ${exp}.vix ${EXP}.vix
  40. checklis.py ${exp}.lis
  41. read -q "REPLY?Are you happy with this lis file? (y/n) "
  42. if [[ ! $REPLY == 'y' ]];then
  43. exit
  44. fi
  45. echo '\n'
  46. getdata.pl -proj ${EXP} -lis ${exp}.lis
  47. j2ms2 -v ${exp}.lis
  48. standardplots -weight ${exp}.ms $2 $3
  49. gv ${exp}-weight.ps
  50. ls ${exp}-auto*ps | parallel 'gv {}'
  51. ls ${exp}-cross*ps | parallel 'gv {}'
  52. ls ${exp}-ampphase*ps | parallel 'gv {}'
  53. read -q "REPLY?Do you want to continue? (y/n) "
  54. if [[ ! $REPLY == 'y' ]];then
  55. exit
  56. fi
  57. echo '\n'
  58. ysfocus.py ${exp}.ms
  59. read "THRESHOLD?Which weight threshold should be applied to the data? tConvert will run later "
  60. flag_weights.py ${exp}.ms $THRESHOLD
  61. read -q "REPLY?Please, update the PI letter. Do you want to continue (tConvert)? (y/n) "
  62. if [[ ! $REPLY == 'y' ]];then
  63. exit
  64. fi
  65. echo '\n'
  66. tConvert ${exp}.ms ${exp}_1_1.IDI
  67. touch ${exp}_${pass}.auth
  68. echo "The experiment has been protected with"
  69. echo "Username: ${exp}"
  70. echo "Password: ${pass}\n"
  71. read -q "REPLY?If you need to PolConvert, DO IT NOW. Do you want to continue and archive? (y/n) "
  72. if [[ ! $REPLY == 'y' ]];then
  73. exit
  74. fi
  75. echo '\n'
  76. gzip *ps
  77. archive -auth -e ${exp}_${date} -n ${exp} -p ${pass}
  78. archive -stnd -e ${exp}_${date} ${exp}.piletter *ps.gz
  79. archive -fits -e ${exp}_${date} *IDI*
  80. pipelet.py ${exp} marcote
  81. echo '\n\nWork at eee finished.\n'
  82. }
  83. function archive_pipeline() {
  84. # First argument should be experiment name (lower cases) second one date (YYMMDD)
  85. ssh jops@jop83 "cd /jop83_0/pipe/in/$1;archive -pipe -e ${1}_${2}"
  86. ssh jops@jop83 "cd /jop83_0/pipe/out/$1;archive -pipe -e ${1}_${2}"
  87. }
  88. function vlbeerexp () {
  89. # Retrieve the log and antabfs files from vlbeer for the given experiment
  90. if [[ ! ( -n $1 && -n $2) ]];then
  91. echo "Two parameters are required:"
  92. echo " - Session (mmmYY e.g. feb18)."
  93. echo " - Experiment name (in lower case)."
  94. else
  95. scp -r evn@vlbeer.ira.inaf.it:vlbi_arch/$1/$2\*log .
  96. scp -r evn@vlbeer.ira.inaf.it:vlbi_arch/$1/$2\*antabfs .
  97. ls
  98. fi
  99. }
  100. function post_process_pipe() {
  101. # Three parameters are expected:
  102. # - The experiment name (case insensitive).
  103. # - The session (in mmmYY format e.g. feb18)
  104. # This script should be run from the experiment folder.
  105. if [[ ! ( -n $1 && -n $2 ) ]];then
  106. echo "Two parameters are required:"
  107. echo " - experiment name (case insensitive)."
  108. echo " - Observing date (in mmmYY format e.g. feb18)."
  109. exit
  110. fi
  111. exp=$1
  112. exp=${(L)1}
  113. EXP=${(U)1}
  114. date=$(ssh jops@ccs grep ${EXP} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d ' ' -f 3)
  115. # In the case of eEVN with an experiment name different this method may not work
  116. if [[ ! -n $date ]];then
  117. date=$(ssh jops@ccs grep ${EXP} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d ' ' -f 4)
  118. fi
  119. # Sometimes it has a \n or empty spaces.
  120. date=${${${date}:s/"\\n"/""}:s/" "/""}
  121. # Create all the required directories and move to marcote/exp one
  122. if [ ! -d $IN/marcote/${exp} ];then
  123. mkdir $IN/marcote/${exp}
  124. fi
  125. if [ ! -d $IN/${exp} ];then
  126. mkdir $IN/${exp}
  127. fi
  128. if [ ! -d $OUT/${exp} ];then
  129. mkdir $OUT/${exp}
  130. fi
  131. cd $IN/marcote/${exp}
  132. vlbeerexp $2 ${exp}
  133. read -q "REPLY?Do you received all ANTAB files? Do you want to create uvflg and antab_check? (y/n) "
  134. if [[ ! $REPLY == 'y' ]];then
  135. exit
  136. fi
  137. echo '\n'
  138. uvflgall.csh
  139. antab_check.py
  140. read -q "REPLY?Have you fixed all ANTAB files? Do you want to continue? (y/n) "
  141. if [[ ! $REPLY == 'y' ]];then
  142. exit
  143. fi
  144. echo '\n'
  145. cat ${exp}*.antabfs > ${exp}.antab
  146. cat ${exp}*.uvflgfs > ${exp}.uvflg
  147. cp ${exp}.antab $IN/${exp}/
  148. cp ${exp}.uvflg $IN/${exp}/
  149. cd $IN/${exp}
  150. # Input file and minimal modifications
  151. cp ../template.inp ${exp}.inp.txt
  152. replace "userno = 3602" "userno = $(give_me_next_userno.sh)" -- ${exp}.inp.txt
  153. replace "experiment = n05c3" "experiment = ${exp}" -- ${exp}.inp.txt
  154. read -q "REPLY?You should now edit the input file... Are you ready to run the EVN pipeline? (y/n) "
  155. if [[ ! $REPLY == 'y' ]];then
  156. exit
  157. fi
  158. echo '\n'
  159. EVN.py ${exp}.inp.txt
  160. read -q "REPLY?Do you want to continue (pipeline properly finished)? (y/n) "
  161. if [[ ! $REPLY == 'y' ]];then
  162. exit
  163. fi
  164. echo '\n'
  165. cd $OUT/${exp}
  166. comment_tasav_file.py ${exp}
  167. feedback.pl -exp "${exp}" -jss 'marcote'
  168. echo "You may need to modify the comment file and/or run again feedback.pl\n"
  169. read -q "REPLY?You may want to archive the pipeline results and protect them afterwards. (y/n) "
  170. if [[ ! $REPLY == 'y' ]];then
  171. exit
  172. fi
  173. echo '\n'
  174. archive_pipeline ${exp} ${date}
  175. ampcal.sh
  176. echo '\n\nWork at pipe finished. You may want to distribute the experiment!\n'
  177. }
  178. if [[ "$(hostname)" = "eee2" ]];then
  179. echo "We are in eee2!"
  180. HOME=/data0/marcote
  181. source /data0/marcote/.zshrc
  182. post_process_eee $1 $2 $3
  183. elif [[ "$(hostname)" = "jop83" ]];then
  184. echo "We are in pipe!"
  185. HOME=/jop83_0/pipe/in/marcote
  186. source /jop83_0/pipe/in/marcote/.zshrc
  187. post_process_pipe $1 $2
  188. echo '\n\nWork at pipe finished. You may want to distribute the experiment!\n'
  189. else
  190. echo "We are somewhere else!"
  191. if [[ ! ( -n $1 && -n $2 && -n $3 && -n $4 ) ]];then
  192. echo "Four parameters are required:"
  193. echo " - experiment name (case insensitive)."
  194. echo " - Reference station to be used for the plots and pipeline."
  195. echo " - Calibrators to be used for the standardplots."
  196. echo " - Observing date (in mmmYY format e.g. jan18)."
  197. exit
  198. fi
  199. # Three parameters are expected:
  200. # - The experiment name (case insensitive).
  201. # - The reference station to use in standardplots.
  202. # - The calibrators to use in standardplots.
  203. # - The session (in mmmYY format e.g. feb18)
  204. ssh -Y jops@eee -t "HOME=/data0/marcote/;zsh -l" "/data0/marcote/scripts/evn_support/post_processing.sh $1 $2 $3"
  205. read -q "REPLY?You may want to continue to pipe. (y/n) "
  206. if [[ ! $REPLY == 'y' ]];then
  207. exit
  208. fi
  209. ssh -Y pipe@jop83 -t "setenv HOME /jop83_0/pipe/in/marcote;zsh -l" "/jop83_0/pipe/in/marcote/scripts/evn_support/post_processing.sh $1 $4"
  210. fi