Collection of scripts and small programs used by the EVN Support Scientists at JIVE during the regular data processing of EVN observations.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

386 lines
16 KiB

  1. #!/usr/bin/env python3
  2. """
  3. Creates the {exp}.comment and {exp}.tasav.txt files for the EVN Pipeline.
  4. Given a default template, customizes it to include the basic data from the given experiment.
  5. Version: 3.0
  6. Date: May 2021
  7. Author: Benito Marcote (marcote@jive.eu)
  8. version 3.0 changes
  9. - Support for passes with primary beam corrections.
  10. version 2.8 changes
  11. - Bug fix happening in some cases for e-EVN (e.g. RSK04).
  12. version 2.7 changes
  13. - Bug fix searching for EXP with _N in MASTER_PROJECTS.LIS
  14. version 2.6 changes
  15. - Bug fix searching for EXP with _N in MASTER_PROJECTS.LIS
  16. version 2.5 changes
  17. - String improvement: now uses 'and' and the end of a list.
  18. version 2.4 changes
  19. - Bug fix when neither target, phaseref, and sources are specified.
  20. version 2.3 changes
  21. - Bug fix reading refant and bandpass.
  22. version 2.2 changes
  23. - Bug fix when phaseref/target are None.
  24. version 2.1 changes
  25. - Fix issue reading MASTER_PROJECTS.LIS.
  26. version 2.0 changes
  27. - Also creates the {exp}.tasav.txt file in the $IN/{exp} directory.
  28. version 1.1 changes
  29. - Now it takes the cutoff value info from the input file.
  30. - Recognises if it is an expected cont/line experiment and change several lines on that.
  31. """
  32. import os
  33. import sys
  34. import argparse
  35. import subprocess
  36. from datetime import datetime as dt
  37. __version__ = 3.0
  38. # The .comment file template is located in the same directory as this script. Or it should be.
  39. template_comment_file = os.path.dirname(os.path.abspath(__file__)) + '/template.comment'
  40. template_tasav_file = os.path.dirname(os.path.abspath(__file__)) + '/template.tasav.txt'
  41. # template-pbcor.tasav.txt is also expected to be at the same location, for primary beam corrected experiments
  42. help_str = """Creates a .comment file (in $OUT/exp directory) and a .tasav.txt file (in $IN/exp).
  43. Given the default templates, it customizes them to include the basic data from the given experiment.
  44. The script takes the information from different locations (ccsbeta, pipe $IN and $OUT directories).
  45. The EVN Pipeline must have been run before calling this script.
  46. """
  47. parser = argparse.ArgumentParser(description=help_str, prog='comment_tasav_file.py')
  48. parser.add_argument('-v', '--version', action='version', version='%(prog)s {}'.format(__version__))
  49. parser.add_argument('-oc', '--output_comment', type=str, default=None, help='Output directory where the file {experiment}.comment will be saved (by default in $OUT/{experiment})')
  50. parser.add_argument('-ot', '--output_tasav', type=str, default=None, help='Output directory where the file {experiment}.tasav.txt will be saved (by default in $IN/{experiment})')
  51. parser.add_argument('experiment', type=str, default=None, help='Experiment name. Note: in case of multiple passes write {exp}_number (e.g. ev100_1')
  52. args = parser.parse_args()
  53. def get_input_file_info():
  54. """Parse the observed sources from the pipeline input file (/jop83_0/pipe/in/{exp}/{exp}.inp.txt).
  55. It searches for the bandpass=, target= and phaseref= lines.
  56. Returns
  57. - refant : str
  58. The reference antenna
  59. - cutoff : int
  60. The SNR cutoff used in FRING.
  61. - bpass : list
  62. The bandpass calibrators and fringe finders used in the pipeline.
  63. - phaseref : list
  64. The phase referencing calibrators used in the pipeline. None if no phase referencing experiment.
  65. - target : list
  66. The targets. Should have the same dimension than the phaseref (unless it is not a phase
  67. referencing experiment).
  68. - do_primarybeam : Bool
  69. If primary beam corrections were applied.
  70. """
  71. with open('/jop83_0/pipe/in/{}/{}.inp.txt'.format(args.experiment.lower().split('_')[0],
  72. args.experiment.lower()), 'r') as inpfile:
  73. phaseref = None
  74. cutoff = 7
  75. target = None
  76. do_primary_beam = False
  77. for inpline in inpfile.readlines():
  78. if 'refant' in inpline and inpline.strip()[0] != '#':
  79. refant = inpline.split('=')[1].strip().split(',')[0]
  80. if ('fring_snr' in inpline) and inpline.strip()[0] != '#':
  81. cutoff = int(inpline.split('=')[1].strip())
  82. if 'bpass' in inpline and inpline.strip()[0] != '#':
  83. bpass = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
  84. if ('phaseref' in inpline) and inpline.strip()[0] != '#':
  85. phaseref = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
  86. if ('target' in inpline) and inpline.strip()[0] != '#':
  87. target = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
  88. if ('sources' in inpline) and inpline.strip()[0] != '#':
  89. if target is None:
  90. target = [i.strip() for i in inpline.split('=')[1].strip().split(',')]
  91. if ('doprimarybeam=1' in inpline.replace(' ', '')) and (inpline.strip()[0] != '#'):
  92. do_primary_beam = True
  93. if target is None:
  94. # No phase referencing experiment and no additional sources specified apart of bpass
  95. target = bpass
  96. return refant, cutoff, bpass, phaseref, target, do_primary_beam
  97. def parse_sources(bpass, phaseref, target):
  98. """Returns the sentences to be placed in the comment file concerning the observed sources
  99. """
  100. s = ''
  101. if phaseref is not None:
  102. assert len(phaseref) == len(target)
  103. for a_phaseref, a_target in zip(phaseref, target):
  104. s += 'The target source {} was calibrated using the phase-reference source {}.<br>\n'.format(
  105. a_target, a_phaseref)
  106. else:
  107. if len(target) > 2:
  108. s += 'The target sources {} were directly fringe-fitted.<br>\n'.format(
  109. ', '.join(target)[::-1].replace(' ,', ' dna ,', 1)[::-1])
  110. elif len(target) == 2:
  111. s += 'The target sources {} were directly fringe-fitted.<br>\n'.format(
  112. ' and '.join(target))
  113. else: # It must be only one target source
  114. assert len(target) == 1
  115. s += 'The target source {} was directly fringe-fitted.<br>\n'.format(target[0])
  116. if target == bpass:
  117. return s
  118. if len(bpass) == 1:
  119. keys = ('was', '')
  120. else:
  121. keys = ('were', 's')
  122. if len(bpass) > 2:
  123. s += '{0} were also observed as calibrators and fringe finders.<br>\n'.format(
  124. ', '.join(bpass)[::-1].replace(' ,', ' dna ,', 1)[::-1])
  125. elif len(bpass) == 2:
  126. s += '{0} were also observed as calibrators and fringe finders.<br>\n'.format(' and '.join(bpass))
  127. else:
  128. assert len(bpass) == 1
  129. s += '{0} was also observed as calibrator and fringe finder.<br>\n'.format(bpass[0])
  130. return s
  131. def get_setup():
  132. """Get the observation setup from the {exp}.SCAN file created by the Pipeline:
  133. It takes the file {exp}.SCAN that should be in /jop83_0/pipe/out/{exp}/.
  134. Returns
  135. - freq : float (GHz)
  136. The central frequency of the observation.
  137. - datarate : float (Mbps)
  138. The datarate of the observation.
  139. - number_ifs : int
  140. Number of IFs or subbands.
  141. - bandwidth : float (MHz)
  142. The bandwidth of each IF or subband.
  143. - pols : int
  144. Number of polarizations:
  145. 1 - single pol.
  146. 2 - dual pol.
  147. 4 - ful pol.
  148. """
  149. with open('/jop83_0/pipe/out/{}/{}.SCAN'.format(args.experiment.lower().split('_')[0],
  150. args.experiment.lower()), 'r') as scanfile:
  151. freq = None
  152. lastline = None
  153. for scanline in scanfile.readlines():
  154. lastline = scanline # It will be the last line at the end of the loop. DO NOT JUDGE ME!!!!
  155. # Getting the frequency and the number of polarizations
  156. # The line is like Freq = XXXX GHz Ncor = X No. vis = XXXX
  157. if 'Freq = ' in scanline:
  158. # freq, pols = [i for i in map([i.strip() for i in scanline.split('=')].__getitem__, ())
  159. temp = ' '.join(scanline.split('=')).split()
  160. freq = float(temp[1])
  161. if temp[2] == 'GHz':
  162. pass
  163. elif temp[2] == 'MHz':
  164. freq *= 1e-3
  165. elif temp[2] == 'kHz':
  166. freq *= 1e-6
  167. elif temp[2] == 'Hz':
  168. freq *= 1e-9
  169. else:
  170. raise ValueError('Not units found in the Freq = XXX line inside the SCAN file')
  171. pols = int(temp[4]) # number of polarizations 2= dual, 4 = full)
  172. assert pols in (1, 2, 4)
  173. if freq is None:
  174. raise IOError('The SCAN file does not contain a line with Freq = XXX')
  175. # The very last line (if not empty) is the last IF with Freq, BW, ch.Sep, and Sideband
  176. last_if = lastline.split()
  177. if len(last_if) == 6:
  178. # It contains the FQID value
  179. number_ifs = int(last_if[1])
  180. bandwidth = int(float(last_if[3])*1e-3)
  181. elif len(last_if) == 5:
  182. # It does not contain the FQID value
  183. number_ifs = int(last_if[0])
  184. bandwidth = int(float(last_if[2])*1e-3)
  185. else:
  186. ValueError('Unexpected number of parameters at the end of the SCAN file.')
  187. if pols == 1:
  188. datarate = number_ifs*bandwidth*2*2
  189. else:
  190. datarate = number_ifs*bandwidth*2*2*2
  191. return freq, datarate, number_ifs, bandwidth, pols
  192. def parse_setup(exp, type_exp, freq, datarate, number_ifs, bandwidth, pols):
  193. """Returns the text to place in the comment file concerning the experiment setup.
  194. Inputs
  195. - exp : str
  196. Experiment name (e.g. n18l2 or EB032). In case of _1, _2, it should have been removed beforehand.
  197. - type_exp : str
  198. To options allowed: 'cont' or 'line', for continuum or spectral line passes, resp.
  199. - freq : float (GHz)
  200. The central frequency of the observation.
  201. - datarate : float (Mbps)
  202. The datarate of the observation.
  203. - number_ifs : int
  204. Number of IFs or subbands.
  205. - bandwidth : float (MHz)
  206. The bandwidth of each IF or subband.
  207. - pols : int
  208. Number of polarizations:
  209. 1 - single pol.
  210. 2 - dual pol.
  211. 4 - ful pol.
  212. """
  213. # It gets the date of the experiment from the MASTER_PROJECTS.LIS file in ccsbeta
  214. date = subprocess.getoutput('ssh jops@ccs grep {} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 3'.format(exp.upper()))
  215. if (date == '') or (date == '\n'):
  216. date = subprocess.getoutput('ssh jops@ccs grep {} /ccs/var/log2vex/MASTER_PROJECTS.LIS | cut -d " " -f 4'.format(exp.upper()))
  217. if '\n' in date:
  218. # Can happen in case of e-EVNs if this is not the official run exp name.
  219. date = date.replace('\n','').strip()
  220. # date = date.split('\n')[-1].strip()
  221. obsdate = dt.strptime(date[-8:], '%Y%m%d')
  222. if freq < 0.6:
  223. band = 'P'
  224. elif freq < 1.9:
  225. band = 'L'
  226. elif freq < 3.0:
  227. band = 'S'
  228. elif freq < 7.0:
  229. band = 'C'
  230. elif freq < 11.0:
  231. band = 'X'
  232. elif freq < 18.0:
  233. band = 'U'
  234. elif freq < 30:
  235. band = 'K'
  236. elif freq >= 30:
  237. band = 'Q'
  238. name_pols = {1: 'single', 2: 'dual', 4: 'full'}
  239. s = '{}. {}-band experiment observed on {}.\n'.format(exp.upper(), band, obsdate.strftime('%d %B %Y'))
  240. s += 'This is a {} pass dataset.<br>\n'.format('continuum' if type_exp == 'cont' else 'spectral line')
  241. s += 'The data rate was {} Mbps ({} x {} MHz subbands, {} polarization, two-bit sampling)<br>\n'.format(
  242. datarate, number_ifs, bandwidth, name_pols[pols])
  243. return s
  244. def get_antennas():
  245. """Returns a list of all antennas participating in the experiment. It takes the information
  246. from the {exp}.DTSUM located in /jop83_0/pipe/out/{exp}/.
  247. """
  248. with open('/jop83_0/pipe/out/{}/{}.DTSUM'.format(args.experiment.lower().split('_')[0],
  249. args.experiment.lower()), 'r') as dtsumfile:
  250. list_antennas = []
  251. inside_array = False
  252. for dtline in dtsumfile.readlines():
  253. if inside_array:
  254. if '(' in dtline:
  255. templine = dtline
  256. # More antennas to get
  257. while '(' in templine:
  258. list_antennas.append(templine[templine.index('(')+1:templine.index(')')].strip())
  259. templine = templine[templine.index(')')+1:]
  260. else:
  261. # We are done
  262. inside_array = False
  263. if 'Array name' in dtline:
  264. inside_array = True
  265. return list_antennas
  266. def parse_antennas(list_antennas):
  267. """Returns the text to include in the comment file concerning the participating antennas
  268. """
  269. list_antennas = [ant.capitalize() for ant in list_antennas]
  270. return '{} stations participated: {}.<br>\n'.format(len(list_antennas), ', '.join(list_antennas))
  271. def parse_line_info(type_exp):
  272. """Places a sentence in the comment file (in different locations) when it is a spectral line pass
  273. to warn that the solutions are expected to be better for continuum passes.
  274. """
  275. if type_exp == 'cont':
  276. return ''
  277. elif type_exp == 'line':
  278. return 'This is the spectral line pass data. Better solutions are expected in the continuum data.'
  279. else:
  280. raise ValueError('Only "cont" or "line" are values expected for type_exp.')
  281. def parse_sources_list(sources, max_item_first_raw=3):
  282. """Converts the list of elements to a comma-separated string list.
  283. The max number of items for the first raw can also be defined.
  284. """
  285. s = ''
  286. if len(sources) > max_item_first_raw:
  287. s += ', '.join(sources[:max_item_first_raw])
  288. sources = sources[max_item_first_raw:]
  289. s += ',\n '
  290. while len(sources) > 6:
  291. s += ', '.join(sources[:6])
  292. s += ',\n '
  293. sources = sources[6:]
  294. s += ', '.join(sources)
  295. return s
  296. refant, fringe_cutoff, *all_sources, do_pb_cor = get_input_file_info()
  297. with open(template_comment_file, 'r') as template:
  298. type_experiment = 'line' if args.experiment[-2:] == '_2' else 'cont'
  299. full_text = template.read()
  300. full_text = full_text.format(setup_header=parse_setup(args.experiment.split('_')[0], type_experiment, *get_setup()),
  301. sources_info=parse_sources(*all_sources),
  302. station_info=parse_antennas(get_antennas()), fringe_cutoff=fringe_cutoff,
  303. ref_antenna=refant, type_info=parse_line_info(type_experiment))
  304. if args.output_comment is None:
  305. outputdir = '/jop83_0/pipe/out/{}'.format(args.experiment.lower().split('_')[0])
  306. else:
  307. outputdir = args.output_comment if args.output_comment[-1] != '/' else args.output_comment[:-1]
  308. comment_file = open('{}/{}.comment'.format(outputdir, args.experiment.lower()), 'w')
  309. comment_file.write(full_text)
  310. comment_file.close()
  311. print('\nFile {0}.comment created successfully in {1}/.'.format(args.experiment.lower(),
  312. outputdir))
  313. refant, fringe_cutoff, bp_sources, pcal_sources, target_sources, do_pb_cor = get_input_file_info()
  314. with open(template_tasav_file.replace('template', 'template-pbcor' if do_pb_cor else 'template'), 'r') as template:
  315. full_text = template.read()
  316. if pcal_sources is None:
  317. full_text = full_text.format(expname=args.experiment.upper().split('_')[0],
  318. fringe_sources=parse_sources_list(bp_sources, 3),
  319. bandpass_sources=parse_sources_list(bp_sources, 4))
  320. else:
  321. full_text = full_text.format(expname=args.experiment.upper().split('_')[0],
  322. fringe_sources=parse_sources_list(list(set(bp_sources + pcal_sources)), 3),
  323. bandpass_sources=parse_sources_list(bp_sources, 4))
  324. if args.output_tasav is None:
  325. outputdir = '/jop83_0/pipe/in/{}'.format(args.experiment.lower().split('_')[0])
  326. else:
  327. outputdir = args.output_tasav if args.output_tasav[-1] != '/' else args.output_tasav[:-1]
  328. comment_file = open('{}/{}.tasav.txt'.format(outputdir, args.experiment.lower()), 'w')
  329. comment_file.write(full_text)
  330. comment_file.close()
  331. print('File {0}.tasav.txt created successfully in {1}/.'.format(args.experiment.lower(), outputdir))