Browse Source

Checks cont/line passes and FRING SNR cutoff

master
Benito Marcote 2 years ago
parent
commit
d5d838f8f1
2 changed files with 70 additions and 29 deletions
  1. +55
    -14
      comment_file.py
  2. +15
    -15
      template.comment

+ 55
- 14
comment_file.py View File

@@ -4,9 +4,13 @@ Creates a .comment file for the EVN Pipeline.
Given a default template, customizes it to include the basic data from the given experiment.
The script will ask you in the terminal about all the required inputs.

Version: 1.0
Version: 1.1
Date: April 2019
Author: Benito Marcote (marcote@jive.eu)

version 1.1 changes
- Now it takes the cutoff value info from the input file.
- Recognises if it is an expected cont/line experiment and change several lines on that.
"""
import os
import sys
@@ -15,7 +19,7 @@ import subprocess
from datetime import datetime as dt


__version__ = 1.0
__version__ = 1.1
# The .comment file template is located in the same directory as this script. Or it should be.
template_file = os.path.dirname(os.path.abspath(__file__)) + '/template.comment'

@@ -34,13 +38,15 @@ args = parser.parse_args()



def get_sources():
def get_input_file_info():
"""Parse the observed sources from the pipeline input file (/jop83_0/pipe/in/{exp}/{exp}.inp.txt).
It searches for the bandpass=, target= and phaseref= lines.

Returns
- refant : str
The reference antenna
- cutoff : int
The SNR cutoff used in FRING.
- bpass : list
The bandpass calibrators and fringe finders used in the pipeline.
- phaseref : list
@@ -52,24 +58,27 @@ def get_sources():
with open('/jop83_0/pipe/in/{}/{}.inp.txt'.format(args.experiment.lower().split('_')[0],
args.experiment.lower()), 'r') as inpfile:
phaseref = None
cutoff = 7
target = None
for inpline in inpfile.readlines():
if 'refant' in inpline:
refant = inpline.split('=')[1].strip().split(',')[0]
if ('fring_snr' in inpline) and inpline[0].strip() != '#':
cutoff = int(inpline.split('=')[1].strip())
if 'bpass' in inpline:
bpass = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
if ('phaseref' in inpline) and inpline[0] != '#':
if ('phaseref' in inpline) and inpline[0].strip() != '#':
phaseref = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
if ('target' in inpline) and inpline[0] != '#':
if ('target' in inpline) and inpline[0].strip() != '#':
target = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
if ('sources' in inpline) and inpline[0] != '#':
if ('sources' in inpline) and inpline[0].strip() != '#':
if target is None:
target = [i.strip() for i in inpline.split('=')[1].strip().split(',')]

if target is None:
raise ValueError('No sources found for target (neither target or sources are defined in INP file')

return refant, bpass, phaseref, target
return refant, cutoff, bpass, phaseref, target


def parse_sources(bpass, phaseref, target):
@@ -167,8 +176,26 @@ def get_setup():
return freq, datarate, number_ifs, bandwidth, pols


def parse_setup(exp, freq, datarate, number_ifs, bandwidth, pols):
def parse_setup(exp, type_exp, freq, datarate, number_ifs, bandwidth, pols):
"""Returns the text to place in the comment file concerning the experiment setup.
Inputs
- exp : str
Experiment name (e.g. n18l2 or EB032).
- type_exp : str
To options allowed: 'cont' or 'line', for continuum or spectral line passes, resp.
- freq : float (GHz)
The central frequency of the observation.
- datarate : float (Mbps)
The datarate of the observation.
- number_ifs : int
Number of IFs or subbands.
- bandwidth : float (MHz)
The bandwidth of each IF or subband.
- pols : int
Number of polarizations:
1 - single pol.
2 - dual pol.
4 - ful pol.
"""
# It gets the date of the experiment from the MASTER_PROJECTS.LIS file in ccsbeta
date = subprocess.getoutput('ssh jops@ccsbeta grep {} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3'.format(exp.upper()))
@@ -191,8 +218,9 @@ def parse_setup(exp, freq, datarate, number_ifs, bandwidth, pols):
band = 'Q'

name_pols = {1: 'single', 2: 'dual', 4: 'full'}
s = '{}. {}-band experiment observed on {}.<br>\n'.format(exp.upper(), band, obsdate.strftime('%d %B %Y'))
s += 'Data rate was {} Mbps ({} x {} MHz subbands, {} polarization, two-bit sampling)<br>\n'.format(
s = '{}. {}-band experiment observed on {}.\n'.format(exp.upper(), band, obsdate.strftime('%d %B %Y'))
s += 'This is the {} pass data.<br>\n'.format('continuum' if type_exp == 'cont' else 'spectral line')
s += 'The data rate was {} Mbps ({} x {} MHz subbands, {} polarization, two-bit sampling)<br>\n'.format(
datarate, number_ifs, bandwidth, name_pols[pols])

return s
@@ -230,13 +258,26 @@ def parse_antennas(list_antennas):
return '{} stations participated: {}.<br>\n'.format(len(list_antennas), ', '.join(list_antennas))


def parse_line_info(type_exp):
"""Places a sentence in the comment file (in different locations) when it is a spectral line pass
to warn that the solutions are expected to be better for continuum passes.
"""
if type_exp == 'cont':
return ''
elif type_exp == 'line':
return 'This is the spectral line pass data. Better solutions are expected in the continuum data.'
else:
raise ValueError('Only "cont" or "line" are values expected for type_exp.')


with open(template_file, 'r') as template:
type_experiment = 'line' if args.experiment[-2:] == '_2' else 'cont'
full_text = template.read()
refant, *all_sources = get_sources()
full_text = full_text.format(setup_header=parse_setup(args.experiment, *get_setup()),
refant, fringe_cutoff, *all_sources = get_input_file_info()
full_text = full_text.format(setup_header=parse_setup(args.experiment, type_experiment, *get_setup()),
sources_info=parse_sources(*all_sources),
station_info=parse_antennas(get_antennas()),
ref_antenna=refant)
station_info=parse_antennas(get_antennas()), fringe_cutoff=fringe_cutoff,
ref_antenna=refant, type_info=parse_line_info(type_experiment))
if args.output is None:
outputdir = '/jop83_0/pipe/out/{}'.format(args.experiment.lower().split('_')[0])
else:


+ 15
- 15
template.comment View File

@@ -4,7 +4,7 @@ GENERAL
{sources_info}<br>
{station_info}<br>

The following are plots from the EVN pipeline analysis, in which the reference antena was: {ref_antenna}.
The following are plots from the EVN pipeline analysis, in which the reference antenna was: {ref_antenna}.
///
POSSM_AUTOCORR
Auto-corr amplitude vs frequency plots for each station showing all IFs and Pols.
@@ -13,33 +13,33 @@ VPLOT_UNCAL
Amplitude and phase vs time, for the whole experiment, no calibration applied. Shows visibilities on all baselines to the reference antenna. All IFs and Pols shown. This is a good place to look for stations that missed scans during the experiment.
///
POSSM_UNCAL
Uncalibrated, vector averaged, cross power specra for baselines to the reference antenna. No bandpass calibration applied yet. This is a good place to look check how well detected sources are on each baseline.
Uncalibrated, vector averaged, cross power specra for baselines to the reference antenna. No bandpass calibration applied yet. This is a good place to check how well detected sources are on each baseline.
///
POSSM_CPOL
///
TSYS
Plots of system temperature vs time for each station (and each IF). Flat-line plots indicate that this information was not available for a given station, in which case we create a generic Tsys table scalled to the station's gain (Jy/K). This information is stored in the TY1 table.
Plots of system temperature vs time for each station (and each IF). Flat-line plots indicate that this information was not available for a given station, in which case we create a generic Tsys table scaled to the station's gain (Jy/K). This information is stored in the TY1 table.
///
GAIN
Gain info in CL2, determined using Tsys tables. Plot shows Gain vs time for each station, each IF and each Pol.
///
FRING_PHAS
Plot of CL3. This is the CL table generated using CL2 (amplitude cal) and SN2 (FRING solutions). Plot shows phase calibration data vs time. Missing data indicate possible issues in those previous steps. If the data are complete (all stations and most scans present) then use this table to phase-reference the target.
Plot of CL3. This is the CL table generated using CL2 (amplitude cal) and SN2 (FRING solutions). Plot shows phase calibration data vs time. Missing data indicate possible issues in those previous steps. If no issues arose, then use this table to phase-reference the target.
///
FRING_DELAY
Plot of SN2. Shows delay vs time from FRING results (usually only calibrators and reference sources). This is a good place to look for times where FRING failed to get solutions. Here FRING uses a signal to noise cutoff of 7. If FRING fails a lot it would be a good idea to retry with a lower signal to noise cutoff. This is the line pass data. We expect better FRING solutions in the continuum data.
Plot of SN2. Shows delay vs time from FRING results (usually only calibrators and reference sources). This is a good place to look for times where FRING failed to get solutions. Here, FRING uses a signal to noise cutoff of {fringe_cutoff}. {type_info}
///
FRING_RATE
SN2 as above, but for rate.
///
BANDPASS
Plot of BP1. Shows the bandpass characteristic of each antenna, determined using the brightest source(s) in the experiment. Look for small dispersion in the phase data, this indicates a high signal to noise - which will lead to better bandpass characterisation.
Plot of BP1. Shows the bandpass characteristics of each antenna, determined using the brightest source(s) in the experiment. Look for small dispersion in the phase data, this indicates a high signal to noise ratio - and thus a good bandpass characterisation.
///
VPLOT_CAL
Plot of visibilities vs time for all baselines to the reference antenna, after applying CL3. This is a good place to look for quickly checking the success of the antab (CL2) and FRING (CL3) stages of the pipeline.
Plot of visibilities vs time for all baselines to the reference antenna, after applying CL3. This is a good place to check the success of the amplitude calibration (CL2) and FRING (CL3) stages of the pipeline.
///
POSSM_CAL
Amplitude and phase vs frequency plots for all baselines to the reference antenna, after applying CL3 and BP1. If CL3 and BP1 are good you will get flat phase and amplitude profiles, therefore this is a good place to check the success of CL2 and particluarly BP1 for each scan/baseline/IF/Pol.
Amplitude and phase vs frequency plots for all baselines to the reference antenna, after applying CL3 and BP1. If the corrections of CL2, CL3 and BP1 are good enough, you will get flat phase and amplitude profiles, therefore this is a good place to check the success of CL2, CL3 and particularly BP1 for each scan/baseline/IF/Pol.
///
IMAPN
Only phase-referenced targets are shown.
@@ -48,26 +48,26 @@ IMAPU
Only phase-referenced targets are shown.
///
CALIB_PHAS1
Individual plots for each source that was used in FRING. Shows phase selfcal solutions (SN1 after SPLIT). This is particularly useful to check if the reference source could be self-calibrated, and to look for missing scans/antennas. These are more useful in the continuum pass data.
Individual plots for each source that was used in FRING. Shows phase selfcal solutions (SN1 after SPLIT). This is particularly useful to check if the reference source could be self-calibrated, and to look for missing scans/antennas. {type_info}
///
CALIB_AMP2
Individual plots for each source that was used in FRING. Shows amplitude selfcal solutions (SN2). This table is important to properly calibrate amplitudes from stations that had to use generic Tsys tables (see TSYS, above). In the statistical summary values close to one means a good a-priori calibration.
Individual plots for each source that was used in FRING. Shows amplitude selfcal solutions (SN2). This table is important to properly calibrate amplitudes from stations that had to use generic Tsys tables (see TSYS, above). In the statistical summary, values close to one mean a good a-priori calibration.
///
SENS
Plot of CL4 which shows amplitude calibrations determined from CL3 and the amplitude selfcal solutions from SN2 (from CALIB_AMP2, above). Comparison with GAIN plot (above) reveals the accuracy of the apriori amplitude calibration derived from the TSYS tables - this is particularly important for stations that have generic generated TSYS tables; Tsys may benefit from aditional scalling if there is a large differnce in SENS and GAIN plots.
Plot of CL4 which shows amplitude calibrations determined from CL3 and the amplitude selfcal solutions from SN2 (from CALIB_AMP2, above). Comparison with GAIN plot (above) reveals the accuracy of the apriori amplitude calibration derived from the TSYS tables - this is particularly important for stations that have generic generated TSYS tables; Tsys may benefit from additional scaling if there is a large difference in SENS and GAIN plots.
///
CLPHS
Closure phases for all triangles of antennas in the array. Individual plots for each source which was used in FRING.
Closure phases for all triangles of antennas in the array. Individual plots for each source that was used in FRING.
///
VPLOT_MODEL
Similar to VPLOT_CAL for sources used in FRING, however each source has been individually SPLITed and modelled.
Similar to VPLOT_CAL for sources used in FRING, however, each source has been individually SPLITed and modelled.
///
UVCOV
Plot of the UV coverage for each source during the experiment.
///
UVPLT
Plot of amplitude vs baseline lenght for each source. This can be useful to look for inications of structure at different angular scales.
Plot of amplitude vs baseline length for each source. This can be useful to look for indications of structure at different angular scales.
///
ICLN
CLEANed image of each source (FRINGed or phase-referened) after applying all relevant calibration by the pipeline up to CL4; amplitude and phase selfcal calibrations.
CLEANed image of each source (FRINGed or phase-referenced) after applying all relevant calibrations up to CL4 by the pipeline; including amplitude and phase self calibrations.
///

Loading…
Cancel
Save