Collection of scripts and small programs used by the EVN Support Scientists at JIVE during the regular data processing of EVN observations.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

60 lines
2.4 KiB

  1. #! /usr/bin/env python
  2. """Checks that all FITS IDI files from the given experiment are accessible
  3. by the EVN Pipeline (they have been archived properly).
  4. This script will avoid a potential problem when the files have been archived
  5. from eee (i.e. moved to the temporary directory in jop83) but still not fully
  6. moved to its final destination in the archive.
  7. If the Pipeline is executed in between those two states, it will run without
  8. showing isses but using only part of the observation.
  9. Version: 1.0
  10. Date: 12 February 2020
  11. Written by Benito Marcote (marcote@jive.eu)
  12. """
  13. import sys
  14. import time
  15. import datetime
  16. import subprocess
  17. def check(expname):
  18. # Get the number of files and sizes from the IDI files in eee
  19. eee_call = ["ssh", "jops@eee", "du", "-ac", f"/data0/\*/{expname.upper()}/\*IDI\*"]
  20. # The output will contain N+1 lines. N files plus the total size in the last one
  21. pipe_call = ["du", "-ac", f"/jop83_1/archive/exp/*{expname.upper()}*/fits/*IDI*"]
  22. # To be done only once
  23. output_eee = subprocess.Popen(' '.join(eee_call), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('utf-8')
  24. files_eee = [outi.split('\t') for outi in output_eee.split('\n') if outi != '']
  25. # [ [ifile_size, ifile_path], ..., [total_size, 'total'] ]
  26. # Loop and sleep in pipe if files not ready
  27. # It waits for intervals of 1, 5, 10, 10, 30, 60, 120 min.
  28. for add_interval in (1, 5, 10, 10, 30, 60, 120):
  29. output_pipe = subprocess.Popen(' '.join(pipe_call), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('utf-8')
  30. files_pipe = [outi.split('\t') for outi in output_pipe.split('\n') if outi != '']
  31. # Check number of files and sizes
  32. # Allow 10% margin as different definitions of 1000/1024..
  33. if len(files_eee) == len(files_pipe):
  34. if abs(float(files_eee[-1][0])/float(files_pipe[-1][0])-1.0) < 0.005:
  35. print('All FITS IDI files are properly archived and the EVN Pipeline will run now.\n')
  36. return True
  37. print('{} - Sleeping for {} min before checking again if files are available.'.format(datetime.datetime.now().strftime('%H:%M'), add_interval))
  38. time.sleep(add_interval*60)
  39. if add_interval == 120:
  40. print('Exiting. Please try to run the EVN Pipeline again in the future.')
  41. return False
  42. if __name__ == '__main__':
  43. check(sys.argv[1].strip())