|
|
|
@ -1,10 +1,10 @@
@@ -1,10 +1,10 @@
|
|
|
|
|
#! /usr/bin/env zsh |
|
|
|
|
#! /bin/zsh |
|
|
|
|
|
|
|
|
|
function post_process_eee() { |
|
|
|
|
# Three parameters are experimentected: |
|
|
|
|
# - The experiment name (case insensitive). |
|
|
|
|
# - The reference station to use in standardplots. |
|
|
|
|
# - The calibrators to use in standardplots. |
|
|
|
|
# Three parameters are expected: |
|
|
|
|
# - The experiment name (case insensitive). |
|
|
|
|
# - The reference station to use in standardplots. |
|
|
|
|
# - The calibrators to use in standardplots. |
|
|
|
|
# This script should be run from the experiment folder. |
|
|
|
|
if [[ ! ( -n $1 && -n $2 && -n $3 ) ]];then |
|
|
|
|
echo "Three parameters are required:" |
|
|
|
@ -14,32 +14,34 @@ function post_process_eee() {
@@ -14,32 +14,34 @@ function post_process_eee() {
|
|
|
|
|
exit |
|
|
|
|
fi |
|
|
|
|
|
|
|
|
|
export PATH=$PATH:/home/jops/scripts/:/home/jops/bin/:/home/jops/.local/bin:$PATH:/data0/marcote/scripts/evn_support/ |
|
|
|
|
experiment=$(echo "${(L)1}") |
|
|
|
|
Experiment=$(echo "${(U)1}") |
|
|
|
|
exp=$1 |
|
|
|
|
exp=${(L)1} |
|
|
|
|
EXP=${(U)1} |
|
|
|
|
|
|
|
|
|
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3) |
|
|
|
|
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3} |
|
|
|
|
# In the case of eEVN with an experiment name different this method may not work |
|
|
|
|
if [[ ! -n $epoch ]];then |
|
|
|
|
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4) |
|
|
|
|
if [[ -n $date ]];then |
|
|
|
|
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4} |
|
|
|
|
fi |
|
|
|
|
|
|
|
|
|
# Sometimes it has a \n or empty spaces. |
|
|
|
|
epoch=${${${epoch}:s/"\\n"/""}:s/" "/""} |
|
|
|
|
epoch=$(echo $epoch | cut -c3-) |
|
|
|
|
date=${${${date}:s/"\\n"/""}:s/" "/""} |
|
|
|
|
|
|
|
|
|
echo "Processing experiment ${Experiment}_${epoch}.\n" |
|
|
|
|
echo 'Processing experiment ${EXP}_${date}.\n' |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Create the lis file from ccs |
|
|
|
|
ssh jops@ccs "cd /ccs/expr/${Experiment};/ccs/bin/make_lis -e ${Experiment} -p prod -s ${experiment}.lis" |
|
|
|
|
scp jops@ccs:/ccs/expr/${Experiment}/${experiment}.vix ./${experiment}.vix |
|
|
|
|
scp jops@ccs:/ccs/expr/${Experiment}/${experiment}.lis ./${experiment}.lis |
|
|
|
|
scp jops@jop83:piletters/${experiment}.piletter . |
|
|
|
|
scp jops@jop83:piletters/${experiment}.expsum . |
|
|
|
|
ln -s ${experiment}.vix ${Experiment}.vix |
|
|
|
|
ssh jops@ccs "cd /ccs/expr/${EXP};/ccs/bin/make_lis -e ${EXP} -p prod -s ${exp}.lis" |
|
|
|
|
|
|
|
|
|
checklis ${experiment}.lis |
|
|
|
|
scp jops@ccs:/ccs/expr/${EXP}/${exp}.vix ./${exp}.vix |
|
|
|
|
scp jops@ccs:/ccs/expr/${EXP}/${exp}.lis ./${exp}.lis |
|
|
|
|
|
|
|
|
|
scp jops@jop83:piletters/${exp}.piletter . |
|
|
|
|
scp jops@jop83:piletters/${exp}.expsum . |
|
|
|
|
|
|
|
|
|
ln -s ${exp}.vix ${EXP}.vix |
|
|
|
|
|
|
|
|
|
checklis ${exp}.lis |
|
|
|
|
|
|
|
|
|
read -q "REPLY?Are you happy with this lis file? (y/n) " |
|
|
|
|
if [[ ! $REPLY == 'y' ]];then |
|
|
|
@ -47,24 +49,17 @@ function post_process_eee() {
@@ -47,24 +49,17 @@ function post_process_eee() {
|
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
getdata.pl -proj ${Experiment} -lis ${experiment}.lis |
|
|
|
|
|
|
|
|
|
j2ms2 -v ${experiment}.lis |
|
|
|
|
|
|
|
|
|
standardplots -weight ${experiment}.ms $2 $3 |
|
|
|
|
getdata.pl -proj ${EXP} -lis ${exp}.lis |
|
|
|
|
|
|
|
|
|
gv ${experiment}-weight.ps |
|
|
|
|
ls ${experiment}-auto*ps | parallel 'gv {}' |
|
|
|
|
ls ${experiment}-cross*ps | parallel 'gv {}' |
|
|
|
|
ls ${experiment}-ampphase*ps | parallel 'gv {}' |
|
|
|
|
j2ms2 -v ${exp}.lis |
|
|
|
|
|
|
|
|
|
standardplots -weight ${exp}.ms $2 $3 |
|
|
|
|
|
|
|
|
|
ysfocus.py ${experiment}.ms |
|
|
|
|
gv ${exp}-weight.ps |
|
|
|
|
ls ${exp}-auto*ps | parallel 'gv {}' |
|
|
|
|
ls ${exp}-cross*ps | parallel 'gv {}' |
|
|
|
|
ls ${exp}-ampphase*ps | parallel 'gv {}' |
|
|
|
|
|
|
|
|
|
read "THRESHOLD?Which weight threshold should be applied to the data? " |
|
|
|
|
# echo '\n' |
|
|
|
|
|
|
|
|
|
flag_weights.py ${experiment}.ms $THRESHOLD |
|
|
|
|
|
|
|
|
|
read -q "REPLY?Please, update the PI letter. Do you want to continue? (y/n) " |
|
|
|
|
if [[ ! $REPLY == 'y' ]];then |
|
|
|
@ -72,10 +67,18 @@ function post_process_eee() {
@@ -72,10 +67,18 @@ function post_process_eee() {
|
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
tConvert ${experiment}.ms ${experiment}_1_1.IDI |
|
|
|
|
|
|
|
|
|
pass=$(date | md5sum | cut -b 1-12) |
|
|
|
|
touch ${experiment}_${pass}.auth |
|
|
|
|
ysfocus.py ${exp}.ms |
|
|
|
|
|
|
|
|
|
read -q "THRESHOLD?Which weight threshold should be applied to the data? " |
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
flag_weights.py ${exp}.ms $THRESHOLD |
|
|
|
|
|
|
|
|
|
tConvert ${exp}.ms ${exp}_1_1.IDI |
|
|
|
|
|
|
|
|
|
export pass=$(date | md5sum | cut -b 1-12) |
|
|
|
|
touch ${exp}_${pass}.auth |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
read -q "REPLY?If you need to PolConvert, DO IT NOW. Do you want to continue? (y/n) " |
|
|
|
@ -85,53 +88,55 @@ function post_process_eee() {
@@ -85,53 +88,55 @@ function post_process_eee() {
|
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
gzip *ps |
|
|
|
|
archive.pl -auth -e ${experiment}_${epoch} -n ${experiment} -p ${pass} |
|
|
|
|
echo "Archiving..." |
|
|
|
|
archive.pl -stnd -e ${experiment}_${epoch} ${experiment}.piletter *ps.gz |
|
|
|
|
archive.pl -fits -e ${experiment}_${epoch} *IDI* |
|
|
|
|
archive -auth -e ${exp}_${date} -n ${exp} -p ${pass} |
|
|
|
|
archive -stnd -e ${exp}_${date} ${exp}.piletter *ps.gz |
|
|
|
|
archive -fits -e ${exp}_${date} *IDI* |
|
|
|
|
|
|
|
|
|
pipelet.py ${experiment} marcote |
|
|
|
|
pipelet.py ${exp} marcote |
|
|
|
|
|
|
|
|
|
echo '\n\nWork at eee finished.\n' |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function archive_pipeline() { |
|
|
|
|
# First argument should be experiment name (lower cases) second one epoch (YYMMDD) |
|
|
|
|
# First argument should be experiment name (lower cases) second one date (YYMMDD) |
|
|
|
|
cd $IN/$1 |
|
|
|
|
su jops -c "/export/jive/jops/bin/archive/user/archive.pl -pipe -e ${1}_${2}" |
|
|
|
|
archive -pipe -e ${1}_${2} |
|
|
|
|
cd $OUT/$1 |
|
|
|
|
su jops -c "/export/jive/jops/bin/archive/user/archive.pl -pipe -e ${1}_${2}" |
|
|
|
|
archive -pipe -e ${1}_${2} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
function post_process_pipe() { |
|
|
|
|
# Three parameters are expected: |
|
|
|
|
# - The experiment name (case insensitive). |
|
|
|
|
# - The reference station to use in standardplots. |
|
|
|
|
# - The calibrators to use in standardplots. |
|
|
|
|
# This script should be run from the experiment folder. |
|
|
|
|
if [[ ! ( -n $1 && -n $2 ) ]];then |
|
|
|
|
echo "Two parameters are required:" |
|
|
|
|
if [[ ! ( -n $1 && -n $2 && -n $3 ) ]];then |
|
|
|
|
echo "Three parameters are required:" |
|
|
|
|
echo " - experiment name (case insensitive)." |
|
|
|
|
echo " - session (in mmmYY format e.g. feb18)." |
|
|
|
|
echo " - Calibrators to be used for the plots." |
|
|
|
|
exit |
|
|
|
|
fi |
|
|
|
|
export PATH=$PATH:/home/jops/scripts/:/home/jops/bin/:/home/jops/.local/bin:$PATH:/data0/marcote/scripts/evn_support/ |
|
|
|
|
source /jop83_0/pipe/in/marcote/.zshrc |
|
|
|
|
experiment=$(echo "${(L)1}") |
|
|
|
|
Experiment=$(echo "${(U)1}") |
|
|
|
|
|
|
|
|
|
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3) |
|
|
|
|
exp=$1 |
|
|
|
|
exp=${(L)1} |
|
|
|
|
EXP=${(U)1} |
|
|
|
|
|
|
|
|
|
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3} |
|
|
|
|
# In the case of eEVN with an experiment name different this method may not work |
|
|
|
|
if [[ ! -n $epoch ]];then |
|
|
|
|
epoch=$(ssh jops@ccs grep $Experiment /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4) |
|
|
|
|
if [[ -n $date ]];then |
|
|
|
|
date=${ssh jops@ccs grep EC067A /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4} |
|
|
|
|
fi |
|
|
|
|
|
|
|
|
|
# Sometimes it has a \n or empty spaces. |
|
|
|
|
epoch=${${${epoch}:s/"\\n"/""}:s/" "/""} |
|
|
|
|
epoch=$(echo $epoch | cut -c3-) |
|
|
|
|
date=${${${date}:s/"\\n"/""}:s/" "/""} |
|
|
|
|
|
|
|
|
|
echo "Processing experiment ${Experiment}_${epoch}.\n" |
|
|
|
|
|
|
|
|
|
# Create all the required directories and move to marcote/experiment one |
|
|
|
|
em ${experiment} |
|
|
|
|
# vlbeerexp $2 ${experiment} |
|
|
|
|
# Create all the required directories and move to marcote/exp one |
|
|
|
|
em ${exp} |
|
|
|
|
vlbeerexp $2 ${exp} |
|
|
|
|
|
|
|
|
|
read -q "REPLY?Do you have all ANTAB files? Do you want to continue? (y/n) " |
|
|
|
|
if [[ ! $REPLY == 'y' ]];then |
|
|
|
@ -139,8 +144,8 @@ function post_process_pipe() {
@@ -139,8 +144,8 @@ function post_process_pipe() {
|
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
# uvflgall.csh |
|
|
|
|
# antab_check.py |
|
|
|
|
uvflgall.csh |
|
|
|
|
antab_check.py |
|
|
|
|
|
|
|
|
|
read -q "REPLY?Have you fixed all ANTAB files? Do you want to continue? (y/n) " |
|
|
|
|
if [[ ! $REPLY == 'y' ]];then |
|
|
|
@ -148,32 +153,27 @@ function post_process_pipe() {
@@ -148,32 +153,27 @@ function post_process_pipe() {
|
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
# cat ${experiment}*.antabfs > "${experiment}.antab" |
|
|
|
|
# cat ${experiment}*.uvflgfs > "${experiment}.uvflg" |
|
|
|
|
# cp "${experiment}.antab" "$IN/${experiment}/" |
|
|
|
|
# cp "${experiment}.uvflg" "$IN/${experiment}/" |
|
|
|
|
cd "$IN/${experiment}" |
|
|
|
|
cat ${exp}*.antabfs > ${exp}.antab |
|
|
|
|
cat ${exp}*.uvflgfs > ${exp}.uvflg |
|
|
|
|
cp ${exp}.antab $IN/${exp}/ |
|
|
|
|
cp ${exp}.uvflg $IN/${exp}/ |
|
|
|
|
cd $IN/${exp} |
|
|
|
|
|
|
|
|
|
# Input file and minimal modifications |
|
|
|
|
cp ../template.inp ${experiment}.inp.txt |
|
|
|
|
replace "userno = 3602" "userno = $(give_me_next_userno.sh)" -- "${experiment}.inp.txt" |
|
|
|
|
replace "experiment = n05c3" "experiment = ${experiment}" -- "${experiment}.inp.txt" |
|
|
|
|
cp ../template.inp ${exp}.inp.txt |
|
|
|
|
replace "userno = 3602" "userno = ${give_me_next_userno.sh}" -- ${exp}.inp.txt |
|
|
|
|
replace "experiment = n05c3" "experiment = ${exp}" -- ${exp}.inp.txt |
|
|
|
|
|
|
|
|
|
read -q "REPLY?You should now edit the input file. Do you want to continue? (y/n) " |
|
|
|
|
if [[ ! $REPLY == 'y' ]];then |
|
|
|
|
exit |
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
EVN.py ${experiment}.inp.txt |
|
|
|
|
echo "You should now edit the input file and run the EVN pipeline by your own.\n" |
|
|
|
|
read -q "REPLY?Do you want to continue (pipeline properly finished)? (y/n) " |
|
|
|
|
if [[ ! $REPLY == 'y' ]];then |
|
|
|
|
exit |
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
|
|
|
|
|
cd $OUT/${experiment} |
|
|
|
|
comment_tasav_file.py ${experiment} |
|
|
|
|
feedback.pl -exp "${experiment}" -jss 'marcote' |
|
|
|
|
cd $OUT/${exp} |
|
|
|
|
comment_tasav_file.py ${exp} |
|
|
|
|
feedback.pl -exp '${exp}' -jss 'marcote' |
|
|
|
|
echo "You may need to modify the comment file and/or run again feedback.pl\n" |
|
|
|
|
|
|
|
|
|
read -q "REPLY?You may want to archive the pipeline results and protect them afterwards. (y/n) " |
|
|
|
@ -181,23 +181,16 @@ function post_process_pipe() {
@@ -181,23 +181,16 @@ function post_process_pipe() {
|
|
|
|
|
exit |
|
|
|
|
fi |
|
|
|
|
echo '\n' |
|
|
|
|
<<<<<<< HEAD |
|
|
|
|
# su jops -c "archive_pipeline ${exp} ${date}" |
|
|
|
|
======= |
|
|
|
|
archive_pipeline ${experiment} ${epoch} |
|
|
|
|
>>>>>>> 8197cb68235ad745473fbdb9c2cf412268d39158 |
|
|
|
|
ampcal.sh |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if [[ `hostname` == "eee2" ]];then |
|
|
|
|
echo "Executing steps from eee..." |
|
|
|
|
post_process_eee $1 $2 $3 |
|
|
|
|
elif [[ `hostname` == "jop83" ]];then |
|
|
|
|
echo "Executing steps from pipe..." |
|
|
|
|
if [[ hostname=="eee2" ]];then |
|
|
|
|
post_process_eee $1 $2 |
|
|
|
|
elif [[ hostname=="jop83" ]];then |
|
|
|
|
post_process_pipe $1 $2 |
|
|
|
|
<<<<<<< HEAD |
|
|
|
|
echo '\n\nWork at pipe finished. You may want to distribute the experiment!\n' |
|
|
|
|
else |
|
|
|
|
# Executing everything from the beginning via ssh. |
|
|
|
@ -205,10 +198,6 @@ else
@@ -205,10 +198,6 @@ else
|
|
|
|
|
ssh pipe@jop83 "post_process_pipe $1 $2" |
|
|
|
|
ssh jops@jop83 "archive_pipeline ${exp} ${date}" |
|
|
|
|
echo '\n\nWork finished. You may want to distribute the experiment!\n' |
|
|
|
|
======= |
|
|
|
|
else |
|
|
|
|
echo "Computer not recognized." |
|
|
|
|
>>>>>>> 8197cb68235ad745473fbdb9c2cf412268d39158 |
|
|
|
|
fi |
|
|
|
|
|
|
|
|
|
|
|
|
|
|