Browse Source

Added ampcal scripts for future developments

master
Benito Marcote 9 months ago
parent
commit
2b19ec1754
2 changed files with 199 additions and 0 deletions
  1. +56
    -0
      ampcal-db.py
  2. +143
    -0
      ampcal.sh

+ 56
- 0
ampcal-db.py View File

@@ -0,0 +1,56 @@
#!/usr/bin/python2
#### ampcal-db.py
#### Jay Blanchard 2017 - python script to take an ampcal.dat file and add each line to the database.


import MySQLdb
import time
from datetime import datetime,timedelta
from warnings import filterwarnings


def insertDB(exp, date, source, telescope, freq, cor):

#basic SQL insert. If the below starts to fail this can be used (HOWEVER DUPLICATE CHECKING WILL HAVE TO BE DONE!)
#sql = """INSERT INTO ampcal(exp, obs_date, source, telescope, frequency, correction) VALUES ('%s', '%s', '%s', '%s', '%s', '%s');""" %(exp, date, source, telescope, freq, cor)

#somewhat complicated SQL here to try to avoid duplicates.
sql = """INSERT INTO ampcal(exp, obs_date, source, telescope, frequency, correction) SELECT * FROM(SELECT '%s', '%s', '%s', '%s', '%s', '%s') AS tmp
WHERE NOT EXISTS (SELECT * FROM ampcal WHERE exp='%s' AND source='%s' AND telescope='%s')
LIMIT 1;""" % (exp, date, source, telescope, freq, cor, exp, source, telescope)

# print sql
try:
cursor.execute(sql)
db.commit()
except (MySQLdb.Error, MySQLdb.Warning) as e:
print(e)
db.rollback()
#MAIN STARTS HERE
#open db con
db = MySQLdb.connect("db0.jive.eu","ampcal-w","klsge[09w4hj","ampcal")
cursor = db.cursor()

#ignore warnings about SQL above...
filterwarnings('ignore', category = MySQLdb.Warning)

print("Inserting ampcal.dat into database")
fin = open('ampcal.dat','rt')
for line in fin:
col = line.split()
year = int(float(col[1]))
rem = float(col[1]) - year
base = datetime(year, 1, 1)
date = base + timedelta(seconds=(base.replace(year=base.year + 1) - base).total_seconds() * rem)
unix_date = int((time.mktime(date.timetuple())+date.microsecond/1000000.0))
try: # neeed to do this in a try for bad rows (easy way to avoid unreadable rows
if (float(col[5]) < 1.0): #we ignore any corrections > 1 as they are likely a bad source
insertDB(col[0],date, col[2], col[3], col[4], col[5])
except:
pass;


fin.close()
#end - close db and file
db.close()

+ 143
- 0
ampcal.sh View File

@@ -0,0 +1,143 @@
#!/bin/bash
###### ampcal.sh
###### Jay Blanchard 2017 - bash script to do ampcal db stuff for an experiment
###### run in the directory for the experiment. Can easily be wrapped for a list of exps etc.
###### needs to pull from ccs to handle jb1/2 so needs ssh keys to be ok.




function readAmpcal {
#$1 is the first argument -> the source for which to get the ampcal
local source=$1
#echo "readAmpcal thinks source is ${source}"

#check if ampcal exists for the source (exp_name will have been defined by the time this is run... scope in bash scripts eh :D)
if [[ -f "${exp_name}_${source}_CALIB_AMP2.ampcal" ]]; then
local amp_file="${exp_name}_${source}_CALIB_AMP2.ampcal"

#else maybe we have a spectral line exp
elif [[ -f "${exp_name}_1_${source}_CALIB_AMP2.ampcal" ]]; then
local amp_file=${exp_name}_1_${source}_CALIB_AMP2.ampcal

#else maybe Benito has named his _0 just to be difficult
elif [[ -f "${exp_name}_0_${source}_CALIB_AMP2.ampcal" ]]; then
local amp_file=${exp_name}_0_${source}_CALIB_AMP2.ampcal


else
echo "There is no ampcal file for calibrator ${source}"
fi


#if file was found!

if [[ -f "${amp_file}" ]]; then
#copy the file first then sed for JB
#I think we no longer care about keeping things separate, we should worry about that when plotting!
cp -puv ${amp_file} $OUT/ampcal/new/
#fix JB
sed -i "s/(JB)/(${jodrell})/g" "$OUT/ampcal/new/${amp_file}"

#need to do ampcaldb as well as update database
#need to do this in that directory because ampcaldb.pl is stupid. note: THIS BROKE EVERYTHING
cd "$OUT/ampcal/new"
ampcaldb.pl "${amp_file}"
cat "ampcal.dat" >> "ampcal.all.dat"
#insert into database:
ampcal-db.py
#have to return to where we were...
cd "-" 1>/dev/null
fi
}



###### START MAIN

#exp name from PWD
exp_name=${PWD##*/}

#getopts because eVLBI is _special_
usage() { echo "Usage: ampcal.sh [-e evlbi_name]"; echo "evlbi_name is the name of the first experiment in the eVLBI session";echo "If your vex file uses non standard naming (not exp.vix) please scp and rename it yourself."; exit 1; }
while getopts ":e:" opt; do
case $opt in
e)
vexName=${OPTARG,,}
echo "eVLBI mode, using vex $vexName.vix"
;;
*)
usage
;;
esac
done



#we need to check we're not in $IN
if [[ "${PWD}" == "${IN}/${exp_name}" ]]; then
echo "You are in ${PWD}! Please run from ${OUT}/${exp_name}"
exit 1
fi

#special handling for JB:
#find out if we are using jb1 or 2
#need to get vix from ccs (if it doesn't exist):
if [[ ! -f "${exp_name}.vix" ]]; then

#if eVLBI we need to copy the evlbi named vix file (we also rename it here so the rest of the code doesn't need changing)
if [[ -n "${vexName}" ]]; then
scp jops@ccs:/ccs/expr/"${vexName^^}"/"${vexName}".vix "${exp_name}".vix
else
scp jops@ccs:/ccs/expr/"${exp_name^^}"/"${exp_name}".vix .
fi
fi

#check if we found it
if [[ ! -f "${exp_name}.vix" ]]; then
echo "Could not find vix. Are you in the experiment directory?"
exit 1
fi

##grep the file for "JODRELL2" if so we are jb2
if [[ -z `grep JODRELL2 ${exp_name}.vix` ]]; then
jodrell="J1"
else
jodrell="J2"
fi

echo "We are ${jodrell}"
#we need to get the list of bandpass cals
#we must cover the case of multi phase centre or spectral line
#first see if $IN/expname.inp.txt exists

if [[ -f "${IN}/${exp_name}/${exp_name}.inp.txt" ]]; then
bpass_str=`grep -h bpass ${IN}/${exp_name}/${exp_name}.inp.txt`
#else check for _1.inp.txt
elif [[ -f "${IN}/${exp_name}/${exp_name}_1.inp.txt" ]]; then
bpass_str=`grep -h bpass ${IN}/${exp_name}/${exp_name}_1.inp.txt`

elif [[ -f "${IN}/${exp_name}/${exp_name}_0.inp.txt" ]]; then
bpass_str=`grep -h bpass ${IN}/${exp_name}/${exp_name}_0.inp.txt`

else
echo "No input file found in ${IN}/${exp_name}! Have you run the pipeline?"
exit 1
fi


if [[ -z bpass_str ]]; then
echo "No bpass found"
exit -1
else
IFS=', ' read -r -a bpass <<< ${bpass_str##*=}

#loop through the list and get the ampcal for each
for cal in "${bpass[@]}"
do
echo ""
echo "Doing calibrator: ${cal}"
readAmpcal "${cal}"
done
fi


Loading…
Cancel
Save