Collection of scripts and small programs used by the EVN Support Scientists at JIVE during the regular data processing of EVN observations.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

177 lines
7.8 KiB

  1. #!/usr/bin/env python3
  2. """Takes the default {exp} file produced by the script and adds the _SRCname to the .cor files.
  3. Generates different {exp} files for the different phase centers (detailed in SRCname).
  4. Version: 2.0
  5. Date: April 2018
  6. Written by Benito Marcote (JIVE;
  7. version 2.0 changes
  8. - Major revision!
  9. - Now you don't need to specify anymore the phase centers to consider
  10. (it had issues when the phase centers do not share a common label)
  11. - It checks the {expname}.lis file to get the jobs that need to be considered.
  12. - It gets the different phase-centers from the existing .cor files.
  13. - Added an option to select a ref station (to include in the j2ms2 line).
  14. - Added an option to exclude some of the phase centers that are in the data.
  15. """
  16. import os
  17. import sys
  18. import glob
  19. import argparse
  20. #from import ascii
  21. help_script = """Create the * script that runs j2ms2 to create a MS file from all the *cor files.
  22. Reads the {expname}.lis file to get a list of all the scans that must be included in the resulting MS.
  23. For each scan checks if there is a {EXPNAME}_NoXXXX.cor* file under the current dir or subdirectory. In case this scan contains information for a multi phase-center, there should be more than one file with that name (called {EXPNAME}_NoXXXX.cor_targetname, where targetname will refer to the different given names for the different phase centers). It will append this cor file to the created *sh file in order to generate a different MS per phase center.
  24. The different MS can optionally contain the information for the calibrators. This information can also go only to one of the phase centers, and the other ones could only contain the target information to save space (see options).
  25. If no parameters are provided (apart of the mandatory, experiment name), then it will include all the sources and phase centers with cor files.
  26. """
  27. help_cals = """Should the calibrators be included in all measurement sets? (in the different phase centers)
  28. By default they are included. If you set this option then they will be removed in all but the first phase center.
  29. """
  30. help_centers = """Phase-center source names. Provide a list (comma-separated, without spaces) of all the phase centers that have been produced and need to be included in the final dataset. NO NEEDED IF --BASE-NAME IS SET.
  31. """
  32. help_exclude = """Phase-center source names to be excluded from the final dataset. Provide a list (comma-separated, without spaces) of those phase centers to be ignored (optional).
  33. """
  34. # help_base_name = """Target name (without phase center suffixes).
  35. # In case of the phase centers share a common prefix in the name of the source (e.g. the different phase centers are src1, src2, src3, ..., srcN), you can just provide the common part ('src' in the previous example) and the script will do the rest. NO NEEDED IF --PHASE-CENTERS IS SET.
  36. # """
  37. help_lisfile = """ (optional) '.lis' file to be loaded. This file is used to get a list of the scans that must be considered for the final MS files. By default it assumes that the file is located in the current directory with the name {expname}.lis. In any other case, this option must be set to localize such file.
  38. """
  39. help_ref_station = """If you want to include the line eo_setup_ref_station:XX in the *sh file to run each j2ms2 line then you must specify here the station name (abreviation).
  40. """
  41. parser = argparse.ArgumentParser(description=help_script)
  42. parser.add_argument('-i', '--include-cals', default=True, dest='cals_on', action='store_false', help=help_cals)
  43. # parser.add_argument('-c', '--phase-centers', type=str, default=None, dest='centers', help=help_centers)
  44. parser.add_argument('-e', '--exclude', type=str, default=None, dest='exclude', help=help_exclude)
  45. parser.add_argument('-l', '--lis', type=str, default=None, dest='lisfile', help=help_lisfile)
  46. # parser.add_argument('-t', '--base-name', type=str, default=None, dest='basename', help=help_base_name)
  47. parser.add_argument('-r', '--ref-station', type=str, default=None, dest='ref_station', help=help_ref_station)
  48. parser.add_argument('expname', type=str, default=None, help='Experiment name')
  49. args = parser.parse_args()
  50. expname = args.expname.lower()
  51. lisfile = expname + '.lis'
  52. if args.lisfile is not None:
  53. lisfile = args.lisfile
  54. # List of all scans to include. Each of it is a string with the format No0000
  55. scans_to_include = []
  56. files_to_include = []
  57. # Max number of SIMULTANEOUS phase centers. That is, in one single pointing.
  58. max_number_of_phase_centers = -1
  59. # Get the list of all the scans that must be considered. Those are selected by the + at the beginning
  60. # of each line in the lis file.
  61. with open(lisfile, 'r') as the_lisfile:
  62. lisfilelines = the_lisfile.readlines()
  63. for lisfileline in lisfilelines:
  64. # Some comments or other non-scan lines, only the ones starting with + are interesting
  65. if lisfileline[0] == '+':
  66. scans_to_include.append(lisfileline.split()[3])
  67. def get_j2ms2_line(outputfile, corfile, ref_station=None):
  68. if ref_station is not None:
  69. return "j2ms2 eo:setup_ref_station={} -o {} {}\n".format(ref_station, outputfile, corfile)
  70. else:
  71. return "j2ms2 -o {} {}\n".format(outputfile, corfile)
  72. if args.exclude is not None:
  73. args.exclude = args.exclude.split(',')
  74. # Get all the cor files (including all phase-centers) ordered by scan
  75. for a_scan in scans_to_include:
  76. scanfiles = glob.glob('./*/*{}.cor*'.format(a_scan))
  77. # Remove the phase centers to exclude
  78. if args.exclude is not None:
  79. for a_scanfile in scanfiles.copy():
  80. if a_scanfile.split('_')[-1] in args.exclude:
  81. scanfiles.remove(a_scanfile)
  82. files_to_include.append(scanfiles)
  83. if len(scanfiles) > max_number_of_phase_centers:
  84. max_number_of_phase_centers = len(scanfiles)
  85. if len(scanfiles) == 0:
  86. print('WARNING: Scan {} listed in the lis file has not been found.'.format(a_scan))
  87. # Create all necessary lists of j2ms2 lines for each phase-center
  88. class Group:
  89. """Simple class that defines a center name and a list of files.
  90. """
  91. def __init__(self):
  92. self._centername = ''
  93. self._centerlist = []
  94. self.files = []
  95. @property
  96. def center(self):
  97. return self._centername
  98. @center.setter
  99. def center(self, name):
  100. # Do not add the name is it is already in the self._centerlist
  101. if name not in self._centerlist:
  102. self._centerlist.append(name)
  103. self._centername = '_'.join(self._centerlist)
  104. # Each group is defined as all the j2ms2 lines that will go to the same output (to the 'center'
  105. # source), by reading all the cor files listed in 'files'
  106. groups_j2ms2 = [Group() for i in range(max_number_of_phase_centers)]
  107. for a_file in files_to_include:
  108. if len(a_file) == 1:
  109. # Include calibrators in all groups or only in the first one?
  110. if args.cals_on:
  111. for a_group in groups_j2ms2:
  112. a_group.files.append(a_file[0])
  113. else:
  114. groups_j2ms2[0].files.append(a_file[0])
  115. else:
  116. # a scan with phase-centers, can be eny length <= max_number_of_phase_centers
  117. for i, a_center in enumerate(sorted(a_file)):
  118. groups_j2ms2[i].files.append(a_center)
  119. groups_j2ms2[i].center = a_center.split('_')[-1]
  120. # Prepare the final sh file!
  121. with open(expname+'', 'w') as shfile:
  122. shfile.write('#!/bin/bash\n')
  123. shfile.write('\\cp {}.vix {}.vix\n'.format(expname, expname.upper()))
  124. for a_group in groups_j2ms2:
  125. shfile.write('echo "Doing phase center {}"\n'.format(
  126. outfile = expname + '.ms_' +
  127. for an_entry in a_group.files:
  128. shfile.write(get_j2ms2_line(outfile, an_entry, args.ref_station))
  129. #os.chmod(a_file, stat.S_IREAD | stat.S_IWUSR | stat.S_IEXEC)
  130. os.chmod(expname+'', 0o755)