marcote
/
evn_support
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

post_procesing now works from any computer

master
Benito Marcote 2 years ago
parent
adadcc1c70
commit
12f3a057cd
1 changed files with 83 additions and 94 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 177
      post_processing.sh

177
post_processing.sh
View File

@ -1,10 +1,10 @@
#! /usr/bin/env zsh
#! /bin/zsh
function post_process_eee() {
# Three parameters are experimentected:
# - The experiment name (case insensitive).
# - The reference station to use in standardplots.
# - The calibrators to use in standardplots.
# Three parameters are expected:
# - The experiment name (case insensitive).
# - The reference station to use in standardplots.
# - The calibrators to use in standardplots.
# This script should be run from the experiment folder.
if [[ ! ( -n $1 && -n $2 && -n $3 ) ]];then
echo "Three parameters are required:"
@ -14,32 +14,34 @@ function post_process_eee() {
exit
fi
export PATH=$PATH:/home/jops/scripts/:/home/jops/bin/:/home/jops/.local/bin:$PATH:/data0/marcote/scripts/evn_support/
experiment=$(echo "${(L)1}")
Experiment=$(echo "${(U)1}")
exp=$1
exp=${(L)1}
EXP=${(U)1}
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3)
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3}
# In the case of eEVN with an experiment name different this method may not work
if [[ ! -n $epoch ]];then
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4)
if [[ -n $date ]];then
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4}
fi
# Sometimes it has a \n or empty spaces.
epoch=${${${epoch}:s/"\\n"/""}:s/" "/""}
epoch=$(echo $epoch | cut -c3-)
date=${${${date}:s/"\\n"/""}:s/" "/""}
echo "Processing experiment ${Experiment}_${epoch}.\n"
echo 'Processing experiment ${EXP}_${date}.\n'
# Create the lis file from ccs
ssh jops@ccs "cd /ccs/expr/${Experiment};/ccs/bin/make_lis -e ${Experiment} -p prod -s ${experiment}.lis"
scp jops@ccs:/ccs/expr/${Experiment}/${experiment}.vix ./${experiment}.vix
scp jops@ccs:/ccs/expr/${Experiment}/${experiment}.lis ./${experiment}.lis
scp jops@jop83:piletters/${experiment}.piletter .
scp jops@jop83:piletters/${experiment}.expsum .
ln -s ${experiment}.vix ${Experiment}.vix
ssh jops@ccs "cd /ccs/expr/${EXP};/ccs/bin/make_lis -e ${EXP} -p prod -s ${exp}.lis"
checklis ${experiment}.lis
scp jops@ccs:/ccs/expr/${EXP}/${exp}.vix ./${exp}.vix
scp jops@ccs:/ccs/expr/${EXP}/${exp}.lis ./${exp}.lis
scp jops@jop83:piletters/${exp}.piletter .
scp jops@jop83:piletters/${exp}.expsum .
ln -s ${exp}.vix ${EXP}.vix
checklis ${exp}.lis
read -q "REPLY?Are you happy with this lis file? (y/n) "
if [[ ! $REPLY == 'y' ]];then
@ -47,24 +49,17 @@ function post_process_eee() {
fi
echo '\n'
getdata.pl -proj ${Experiment} -lis ${experiment}.lis
j2ms2 -v ${experiment}.lis
standardplots -weight ${experiment}.ms $2 $3
getdata.pl -proj ${EXP} -lis ${exp}.lis
gv ${experiment}-weight.ps
ls ${experiment}-auto*ps | parallel 'gv {}'
ls ${experiment}-cross*ps | parallel 'gv {}'
ls ${experiment}-ampphase*ps | parallel 'gv {}'
j2ms2 -v ${exp}.lis
standardplots -weight ${exp}.ms $2 $3
ysfocus.py ${experiment}.ms
gv ${exp}-weight.ps
ls ${exp}-auto*ps | parallel 'gv {}'
ls ${exp}-cross*ps | parallel 'gv {}'
ls ${exp}-ampphase*ps | parallel 'gv {}'
read "THRESHOLD?Which weight threshold should be applied to the data? "
# echo '\n'
flag_weights.py ${experiment}.ms $THRESHOLD
read -q "REPLY?Please, update the PI letter. Do you want to continue? (y/n) "
if [[ ! $REPLY == 'y' ]];then
@ -72,10 +67,18 @@ function post_process_eee() {
fi
echo '\n'
tConvert ${experiment}.ms ${experiment}_1_1.IDI
pass=$(date | md5sum | cut -b 1-12)
touch ${experiment}_${pass}.auth
ysfocus.py ${exp}.ms
read -q "THRESHOLD?Which weight threshold should be applied to the data? "
echo '\n'
flag_weights.py ${exp}.ms $THRESHOLD
tConvert ${exp}.ms ${exp}_1_1.IDI
export pass=$(date | md5sum | cut -b 1-12)
touch ${exp}_${pass}.auth
read -q "REPLY?If you need to PolConvert, DO IT NOW. Do you want to continue? (y/n) "
@ -85,53 +88,55 @@ function post_process_eee() {
echo '\n'
gzip *ps
archive.pl -auth -e ${experiment}_${epoch} -n ${experiment} -p ${pass}
echo "Archiving..."
archive.pl -stnd -e ${experiment}_${epoch} ${experiment}.piletter *ps.gz
archive.pl -fits -e ${experiment}_${epoch} *IDI*
archive -auth -e ${exp}_${date} -n ${exp} -p ${pass}
archive -stnd -e ${exp}_${date} ${exp}.piletter *ps.gz
archive -fits -e ${exp}_${date} *IDI*
pipelet.py ${experiment} marcote
pipelet.py ${exp} marcote
echo '\n\nWork at eee finished.\n'
}
function archive_pipeline() {
# First argument should be experiment name (lower cases) second one epoch (YYMMDD)
# First argument should be experiment name (lower cases) second one date (YYMMDD)
cd $IN/$1
su jops -c "/export/jive/jops/bin/archive/user/archive.pl -pipe -e ${1}_${2}"
archive -pipe -e ${1}_${2}
cd $OUT/$1
su jops -c "/export/jive/jops/bin/archive/user/archive.pl -pipe -e ${1}_${2}"
archive -pipe -e ${1}_${2}
}
function post_process_pipe() {
# Three parameters are expected:
# - The experiment name (case insensitive).
# - The reference station to use in standardplots.
# - The calibrators to use in standardplots.
# This script should be run from the experiment folder.
if [[ ! ( -n $1 && -n $2 ) ]];then
echo "Two parameters are required:"
if [[ ! ( -n $1 && -n $2 && -n $3 ) ]];then
echo "Three parameters are required:"
echo " - experiment name (case insensitive)."
echo " - session (in mmmYY format e.g. feb18)."
echo " - Calibrators to be used for the plots."
exit
fi
export PATH=$PATH:/home/jops/scripts/:/home/jops/bin/:/home/jops/.local/bin:$PATH:/data0/marcote/scripts/evn_support/
source /jop83_0/pipe/in/marcote/.zshrc
experiment=$(echo "${(L)1}")
Experiment=$(echo "${(U)1}")
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3)
exp=$1
exp=${(L)1}
EXP=${(U)1}
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3}
# In the case of eEVN with an experiment name different this method may not work
if [[ ! -n $epoch ]];then
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4)
if [[ -n $date ]];then
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4}
fi
# Sometimes it has a \n or empty spaces.
epoch=${${${epoch}:s/"\\n"/""}:s/" "/""}
epoch=$(echo $epoch | cut -c3-)
date=${${${date}:s/"\\n"/""}:s/" "/""}
echo "Processing experiment ${Experiment}_${epoch}.\n"
# Create all the required directories and move to marcote/experiment one
em ${experiment}
# vlbeerexp $2 ${experiment}
# Create all the required directories and move to marcote/exp one
em ${exp}
vlbeerexp $2 ${exp}
read -q "REPLY?Do you have all ANTAB files? Do you want to continue? (y/n) "
if [[ ! $REPLY == 'y' ]];then
@ -139,8 +144,8 @@ function post_process_pipe() {
fi
echo '\n'
# uvflgall.csh
# antab_check.py
uvflgall.csh
antab_check.py
read -q "REPLY?Have you fixed all ANTAB files? Do you want to continue? (y/n) "
if [[ ! $REPLY == 'y' ]];then
@ -148,32 +153,27 @@ function post_process_pipe() {
fi
echo '\n'
# cat ${experiment}*.antabfs > "${experiment}.antab"
# cat ${experiment}*.uvflgfs > "${experiment}.uvflg"
# cp "${experiment}.antab" "$IN/${experiment}/"
# cp "${experiment}.uvflg" "$IN/${experiment}/"
cd "$IN/${experiment}"
cat ${exp}*.antabfs > ${exp}.antab
cat ${exp}*.uvflgfs > ${exp}.uvflg
cp ${exp}.antab $IN/${exp}/
cp ${exp}.uvflg $IN/${exp}/
cd $IN/${exp}
# Input file and minimal modifications
cp ../template.inp ${experiment}.inp.txt
replace "userno = 3602" "userno = $(give_me_next_userno.sh)" -- "${experiment}.inp.txt"
replace "experiment = n05c3" "experiment = ${experiment}" -- "${experiment}.inp.txt"
cp ../template.inp ${exp}.inp.txt
replace "userno = 3602" "userno = ${give_me_next_userno.sh}" -- ${exp}.inp.txt
replace "experiment = n05c3" "experiment = ${exp}" -- ${exp}.inp.txt
read -q "REPLY?You should now edit the input file. Do you want to continue? (y/n) "
if [[ ! $REPLY == 'y' ]];then
exit
fi
echo '\n'
EVN.py ${experiment}.inp.txt
echo "You should now edit the input file and run the EVN pipeline by your own.\n"
read -q "REPLY?Do you want to continue (pipeline properly finished)? (y/n) "
if [[ ! $REPLY == 'y' ]];then
exit
fi
echo '\n'
cd $OUT/${experiment}
comment_tasav_file.py ${experiment}
feedback.pl -exp "${experiment}" -jss 'marcote'
cd $OUT/${exp}
comment_tasav_file.py ${exp}
feedback.pl -exp '${exp}' -jss 'marcote'
echo "You may need to modify the comment file and/or run again feedback.pl\n"
read -q "REPLY?You may want to archive the pipeline results and protect them afterwards. (y/n) "
@ -181,23 +181,16 @@ function post_process_pipe() {
exit
fi
echo '\n'
<<<<<<< HEAD
# su jops -c "archive_pipeline ${exp} ${date}"
=======
archive_pipeline ${experiment} ${epoch}
>>>>>>> 8197cb68235ad745473fbdb9c2cf412268d39158
ampcal.sh
}
if [[ `hostname` == "eee2" ]];then
echo "Executing steps from eee..."
post_process_eee $1 $2 $3
elif [[ `hostname` == "jop83" ]];then
echo "Executing steps from pipe..."
if [[ hostname=="eee2" ]];then
post_process_eee $1 $2
elif [[ hostname=="jop83" ]];then
post_process_pipe $1 $2
<<<<<<< HEAD
echo '\n\nWork at pipe finished. You may want to distribute the experiment!\n'
else
# Executing everything from the beginning via ssh.
@ -205,10 +198,6 @@ else
ssh pipe@jop83 "post_process_pipe $1 $2"
ssh jops@jop83 "archive_pipeline ${exp} ${date}"
echo '\n\nWork finished. You may want to distribute the experiment!\n'
=======
else
echo "Computer not recognized."
>>>>>>> 8197cb68235ad745473fbdb9c2cf412268d39158
fi

Write Preview
Loading…
Cancel
Save