The vbs tools - vbs_ls, vbs_rm, vbs_fs - for listing, removing and mounting vbs and Mark6 format scattered VLBI recordings on FlexBuff and Mark6 systems
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

185 lines
10 KiB

  1. #!/usr/bin/env python
  2. # Script to remove flexbuff/Mark6 recording(s)
  3. import os, re, sys, glob, copy, struct, fnmatch, argparse, operator, itertools, functools, collections
  4. version = "$Id$"
  5. flexbuff_pattern = '/mnt/disk*'
  6. mark6_pattern = '/mnt/disks/*/*/data'
  7. vbs_chunk = lambda recording: re.compile(r"^"+re.escape(recording)+r"\.[0-9]{8}$")
  8. description = """Remove FlexBuff/Mark6 recording(s) from the drives, much like rm(1), only specialized for scattered recordings"""
  9. compose = lambda *funcs: lambda x: reduce(lambda v, f: f(v), reversed(funcs), x)
  10. choice = lambda pred, t, f: lambda x: t(x) if pred(x) else f(x)
  11. append = lambda l, e: l.append(e) or l
  12. method = lambda f : lambda *args: f(*args)
  13. mk = lambda **kwargs: type('', (), {'__init__':lambda o: functools.reduce(lambda a, av: setattr(a, av[0], copy.deepcopy(av[1])) or a, kwargs.iteritems(), o).__dict__.update()})
  14. mk_rm_obj = lambda mp, p, **kwargs: mk(path=os.path.join(mp, p), recording=p, **kwargs)()
  15. # Partition a list into two lists: matching and non-matching
  16. def partition(pred, l):
  17. (yes, no) = ([], [])
  18. for item in l:
  19. yes.append(item) if pred(item) else no.append(item)
  20. return (yes, no)
  21. # Mk6 file header layout
  22. # uint32_t sync_word; // MARK6_SG_SYNC_WORD = 0xfeed6666
  23. # int32_t version; // defines format of file
  24. # int32_t block_size; // length of blocks including header (bytes)
  25. # int32_t packet_format; // format of data packets, enumerated below
  26. # int32_t packet_size; // length of packets (bytes)
  27. mk_obj = lambda **kwargs: type('', (), kwargs)()
  28. mk6_hdr_f = '<5I'
  29. mk6_hdr_sz = struct.calcsize(mk6_hdr_f)
  30. is_mk6 = compose(choice(lambda x: len(x) == mk6_hdr_sz,
  31. compose(lambda y: y[0] == 0xfeed6666 and y[1] == 2, functools.partial(struct.unpack, mk6_hdr_f)),
  32. lambda z: None),
  33. operator.methodcaller('read', mk6_hdr_sz), open)
  34. # mp_patterns = (mountpoint, [pattern [,pattern, ...]])
  35. # => scan the indicated mountpoint for recordings that match any of the patterns
  36. def index(acc, mp_patterns):
  37. (mountpoint, patterns) = mp_patterns
  38. # process unique set of file, dir entries even if multiple patterns match
  39. for (rec, path) in map(lambda match: (match, os.path.join(mountpoint, match)),
  40. reduce(lambda b, pattern: b.update(fnmatch.filter(os.listdir(mountpoint), pattern)) or b,
  41. patterns, set())):
  42. # Directories under mountpoints are possible vbs recordings,
  43. if os.path.isdir(path):
  44. # figure out if there is at least one entry of the form "maybe_rec/maybe_rec.XXXXXXXX"
  45. # Note that we do *not* compile a full list of all chunks. If deleting a lot of
  46. # recordings the mem'ry consumption might get unwieldy
  47. contents = os.listdir(path)
  48. acc[rec].dirs.append(mk_rm_obj(mountpoint, rec, is_empty=not contents,
  49. is_vbs=next(itertools.dropwhile(compose(operator.not_, vbs_chunk(rec).match), contents), None)))
  50. # files may be Mk6 recordings
  51. elif os.path.isfile(path) and is_mk6(path):
  52. acc[rec].files.append(mk_rm_obj(mountpoint, rec))
  53. return acc
  54. # Functions to remove a directory or a file
  55. def remove_dir(entry):
  56. # First remove all chunks, then try to rmdir the entry
  57. (chunks, gunk) = partition(vbs_chunk(entry.recording).match, os.listdir(entry.path))
  58. map(os.unlink, map(functools.partial(os.path.join, entry.path), chunks))
  59. os.rmdir( entry.path ) if not gunk else None
  60. remove_file = compose(os.unlink, operator.attrgetter('path'))
  61. # Functions which ask for confirmation or not. You get ~1000 tries to answer an acceptable answer ...
  62. dont_ask = lambda x: (dont_ask, True)
  63. skip = lambda x: (skip, False)
  64. def ask(x):
  65. ans = (raw_input("{0}? [N/y/a/q] ".format(x)) or "n").lower()
  66. return (ask, True) if ans == "y" else \
  67. ((ask, False) if ans == "n" else \
  68. ((dont_ask, True) if ans == "a" else \
  69. ((skip, False) if ans == "q" else ask(x))))
  70. # return True if the entry contains anything that can be removed
  71. # entry is a tuple of (recording, object.{files, dirs})
  72. def maybe_remove(entry):
  73. (empty, vbs) = zip(*map(operator.attrgetter('is_empty', 'is_vbs'), entry[1].dirs))
  74. # check dirs - only OK to remove if all empty or at least one has detected VBS
  75. # note: have to add check for non-empty dirs because all([]) == True ...
  76. # making "all([]) or any([])" evaluate to True which would be Wrong (tm)
  77. return bool(entry[1].files or (entry[1].dirs and (all(empty) or any(vbs))))
  78. # entry is a tuple of (recording, object.{files, dirs}), and we know either files and/or dirs is non-empty
  79. def remove(ask_fn, entry):
  80. summary = ', '.join(map(functools.partial(str.format, "{0[0]} {0[1]}"),
  81. filter(operator.itemgetter(0), zip(map(len, [entry[1].files, entry[1].dirs]), ["files", "dirs"]))))
  82. (ask_fn, remove_it) = ask_fn( entry[0]+" ["+summary+"]" )
  83. if remove_it:
  84. try:
  85. map(remove_file, entry[1].files)
  86. map(remove_dir, entry[1].dirs)
  87. except OSError, E:
  88. print entry[0],": ",E.strerror
  89. return ask_fn
  90. ###########################################################################
  91. #
  92. # Command line parsing
  93. #
  94. ###########################################################################
  95. # 'append_list' action: a helper to append a list of items to a variable
  96. # i.e. to support multiple '-R pattern1 -R pattern2 ...' options
  97. # Note that we only accept 'nargs' values that will actually result inna list:
  98. # nargs = '+' or '*', int >= 0, argparse.REMAINDER
  99. class AppendList(argparse.Action):
  100. def __init__(self, option_strings, dest, nargs=None, **kwargs):
  101. super(AppendList, self).__init__(option_strings, dest, nargs=nargs, **kwargs)
  102. if not ((isinstance(nargs, str) and len(nargs) == 1 and nargs[0] in '+*') or
  103. (isinstance(nargs, int) and nargs >= 0) or
  104. (nargs == argparse.REMAINDER)):
  105. raise RuntimeError, "nargs must be '+', '*', an integer >= 0 or argparse.REMAINDER for this action"
  106. def __call__(self, p, ns, values, option_string):
  107. setattr(ns, self.dest, values) if not hasattr(ns, self.dest) else setattr(ns, self.dest, getattr(ns, self.dest)+values)
  108. class PrintHelp(argparse.Action):
  109. def __call__(self, p, *args):
  110. p.print_help() or sys.exit(0)
  111. parsert = argparse.ArgumentParser(description=description, add_help=False)
  112. parsert.add_argument('--help', nargs=0, action=PrintHelp, help="show this help message and exit succesfully")
  113. parsert.add_argument('-6', dest='rootDirs', action='append_const', default=[], const=mark6_pattern,
  114. help="Look for recordings in Mark6 mountpoints")
  115. parsert.add_argument('-v', dest='rootDirs', action='append_const', default=[], const=flexbuff_pattern,
  116. help="Look for recordings in FlexBuff mountpoints (default)")
  117. parsert.add_argument('-f', action="store_const", default=ask, dest='confirmation', const=dont_ask,
  118. help="Don't ask for confirmation, just remove. Overrides any previous '-i' option(s). Use (1) with caution and (2) at own risk")
  119. parsert.add_argument('-i', action="store_const", default=ask, dest='confirmation', const=ask,
  120. help="Always ask for confirmation. Overrides any previous '-f' option(s)")
  121. parsert.add_argument('--watch-the-blood-dripping', default=None, action="store_true", dest="watch_the_blood_dripping",
  122. help=argparse.SUPPRESS)
  123. parsert.add_argument("-R", nargs=1, action=AppendList, dest='rootDirs',
  124. help="Append directories matching the pattern to the vbs_rm search path. Shell-style wildcards ('*?') are supported. This option may be present multiple times to add multiple patterns")
  125. parsert.add_argument('--version', action='version', version=version, help="Print current version and exit succesfully")
  126. parsert.add_argument("pattern", nargs='+',
  127. help="Delete recordings matching these pattern(s). Shell-style wildcards ('*?') are supported.")
  128. # deal with command line
  129. userinput = parsert.parse_args()
  130. # If there is a pattern that is built of only wildcards [i.e. which would match all recordings, i.e. a request to
  131. # delete all recordings ...] we'll ask the user to confirm that this is his/her true intent
  132. if filter(re.compile(r'^((\*[\*\?]*)|([\*\?]*\*)|(\?+\*[\*\?]*))$').match, userinput.pattern) and \
  133. not (userinput.confirmation is dont_ask and userinput.watch_the_blood_dripping is True):
  134. print "###############################################################"
  135. print
  136. print " You've requested to remove ALL FlexBuff/Mark6 recordings"
  137. print
  138. print " For this program to accept this and proceed, you MUST run it"
  139. print " with the following command line flags, to indicate you've "
  140. print " understood the possible consequences:"
  141. print
  142. print " $> vbs_rm -f --watch-the-blood-dripping *"
  143. print
  144. print
  145. print " >>>> You might want to run 'vbs_ls -l' first to <<<<"
  146. print " >>>> get an overview of what you're about to delete <<<<"
  147. print
  148. print
  149. print "###############################################################"
  150. sys.exit(1)
  151. ####################################################################################
  152. # Analyze what the user actually wanted
  153. ####################################################################################
  154. # 1.) Get the list of mountpoints
  155. mountpoints = reduce(lambda a, pattern: a.update(filter(lambda p: os.path.isdir(p) and os.access(p, os.X_OK|os.R_OK),
  156. glob.glob(pattern))) or a,
  157. userinput.rootDirs or [flexbuff_pattern], set())
  158. # 2.) Zip all mountpoints with all pattern(s) and collect all matching recordings. For each entry split into list of files and list of dirs.
  159. # Filter out recordings that have anything to remove at all and then do that
  160. reduce(remove, filter(maybe_remove,
  161. reduce(index, zip(mountpoints, itertools.repeat(userinput.pattern)), collections.defaultdict(mk(files=list(), dirs=list()))).iteritems()
  162. ), userinput.confirmation)