The vbs tools - vbs_ls, vbs_rm, vbs_fs - for listing, removing and mounting vbs and Mark6 format scattered VLBI recordings on FlexBuff and Mark6 systems
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

184 lines
10 KiB

#!/usr/bin/env python
# Script to remove flexbuff/Mark6 recording(s)
import os, re, sys, glob, copy, struct, fnmatch, argparse, operator, itertools, functools, collections
version = "$Id$"
flexbuff_pattern = '/mnt/disk*'
mark6_pattern = '/mnt/disks/*/*/data'
vbs_chunk = lambda recording: re.compile(r"^"+re.escape(recording)+r"\.[0-9]{8}$")
description = """Remove FlexBuff/Mark6 recording(s) from the drives, much like rm(1), only specialized for scattered recordings"""
compose = lambda *funcs: lambda x: reduce(lambda v, f: f(v), reversed(funcs), x)
choice = lambda pred, t, f: lambda x: t(x) if pred(x) else f(x)
append = lambda l, e: l.append(e) or l
method = lambda f : lambda *args: f(*args)
mk = lambda **kwargs: type('', (), {'__init__':lambda o: functools.reduce(lambda a, av: setattr(a, av[0], copy.deepcopy(av[1])) or a, kwargs.iteritems(), o).__dict__.update()})
mk_rm_obj = lambda mp, p, **kwargs: mk(path=os.path.join(mp, p), recording=p, **kwargs)()
# Partition a list into two lists: matching and non-matching
def partition(pred, l):
(yes, no) = ([], [])
for item in l:
yes.append(item) if pred(item) else no.append(item)
return (yes, no)
# Mk6 file header layout
# uint32_t sync_word; // MARK6_SG_SYNC_WORD = 0xfeed6666
# int32_t version; // defines format of file
# int32_t block_size; // length of blocks including header (bytes)
# int32_t packet_format; // format of data packets, enumerated below
# int32_t packet_size; // length of packets (bytes)
mk_obj = lambda **kwargs: type('', (), kwargs)()
mk6_hdr_f = '<5I'
mk6_hdr_sz = struct.calcsize(mk6_hdr_f)
is_mk6 = compose(choice(lambda x: len(x) == mk6_hdr_sz,
compose(lambda y: y[0] == 0xfeed6666 and y[1] == 2, functools.partial(struct.unpack, mk6_hdr_f)),
lambda z: None),
operator.methodcaller('read', mk6_hdr_sz), open)
# mp_patterns = (mountpoint, [pattern [,pattern, ...]])
# => scan the indicated mountpoint for recordings that match any of the patterns
def index(acc, mp_patterns):
(mountpoint, patterns) = mp_patterns
# process unique set of file, dir entries even if multiple patterns match
for (rec, path) in map(lambda match: (match, os.path.join(mountpoint, match)),
reduce(lambda b, pattern: b.update(fnmatch.filter(os.listdir(mountpoint), pattern)) or b,
patterns, set())):
# Directories under mountpoints are possible vbs recordings,
if os.path.isdir(path):
# figure out if there is at least one entry of the form "maybe_rec/maybe_rec.XXXXXXXX"
# Note that we do *not* compile a full list of all chunks. If deleting a lot of
# recordings the mem'ry consumption might get unwieldy
contents = os.listdir(path)
acc[rec].dirs.append(mk_rm_obj(mountpoint, rec, is_empty=not contents,
is_vbs=next(itertools.dropwhile(compose(operator.not_, vbs_chunk(rec).match), contents), None)))
# files may be Mk6 recordings
elif os.path.isfile(path) and is_mk6(path):
acc[rec].files.append(mk_rm_obj(mountpoint, rec))
return acc
# Functions to remove a directory or a file
def remove_dir(entry):
# First remove all chunks, then try to rmdir the entry
(chunks, gunk) = partition(vbs_chunk(entry.recording).match, os.listdir(entry.path))
map(os.unlink, map(functools.partial(os.path.join, entry.path), chunks))
os.rmdir( entry.path ) if not gunk else None
remove_file = compose(os.unlink, operator.attrgetter('path'))
# Functions which ask for confirmation or not. You get ~1000 tries to answer an acceptable answer ...
dont_ask = lambda x: (dont_ask, True)
skip = lambda x: (skip, False)
def ask(x):
ans = (raw_input("{0}? [N/y/a/q] ".format(x)) or "n").lower()
return (ask, True) if ans == "y" else \
((ask, False) if ans == "n" else \
((dont_ask, True) if ans == "a" else \
((skip, False) if ans == "q" else ask(x))))
# return True if the entry contains anything that can be removed
# entry is a tuple of (recording, object.{files, dirs})
def maybe_remove(entry):
(empty, vbs) = zip(*map(operator.attrgetter('is_empty', 'is_vbs'), entry[1].dirs))
# check dirs - only OK to remove if all empty or at least one has detected VBS
# note: have to add check for non-empty dirs because all([]) == True ...
# making "all([]) or any([])" evaluate to True which would be Wrong (tm)
return bool(entry[1].files or (entry[1].dirs and (all(empty) or any(vbs))))
# entry is a tuple of (recording, object.{files, dirs}), and we know either files and/or dirs is non-empty
def remove(ask_fn, entry):
summary = ', '.join(map(functools.partial(str.format, "{0[0]} {0[1]}"),
filter(operator.itemgetter(0), zip(map(len, [entry[1].files, entry[1].dirs]), ["files", "dirs"]))))
(ask_fn, remove_it) = ask_fn( entry[0]+" ["+summary+"]" )
if remove_it:
try:
map(remove_file, entry[1].files)
map(remove_dir, entry[1].dirs)
except OSError, E:
print entry[0],": ",E.strerror
return ask_fn
###########################################################################
#
# Command line parsing
#
###########################################################################
# 'append_list' action: a helper to append a list of items to a variable
# i.e. to support multiple '-R pattern1 -R pattern2 ...' options
# Note that we only accept 'nargs' values that will actually result inna list:
# nargs = '+' or '*', int >= 0, argparse.REMAINDER
class AppendList(argparse.Action):
def __init__(self, option_strings, dest, nargs=None, **kwargs):
super(AppendList, self).__init__(option_strings, dest, nargs=nargs, **kwargs)
if not ((isinstance(nargs, str) and len(nargs) == 1 and nargs[0] in '+*') or
(isinstance(nargs, int) and nargs >= 0) or
(nargs == argparse.REMAINDER)):
raise RuntimeError, "nargs must be '+', '*', an integer >= 0 or argparse.REMAINDER for this action"
def __call__(self, p, ns, values, option_string):
setattr(ns, self.dest, values) if not hasattr(ns, self.dest) else setattr(ns, self.dest, getattr(ns, self.dest)+values)
class PrintHelp(argparse.Action):
def __call__(self, p, *args):
p.print_help() or sys.exit(0)
parsert = argparse.ArgumentParser(description=description, add_help=False)
parsert.add_argument('--help', nargs=0, action=PrintHelp, help="show this help message and exit succesfully")
parsert.add_argument('-6', dest='rootDirs', action='append_const', default=[], const=mark6_pattern,
help="Look for recordings in Mark6 mountpoints")
parsert.add_argument('-v', dest='rootDirs', action='append_const', default=[], const=flexbuff_pattern,
help="Look for recordings in FlexBuff mountpoints (default)")
parsert.add_argument('-f', action="store_const", default=ask, dest='confirmation', const=dont_ask,
help="Don't ask for confirmation, just remove. Overrides any previous '-i' option(s). Use (1) with caution and (2) at own risk")
parsert.add_argument('-i', action="store_const", default=ask, dest='confirmation', const=ask,
help="Always ask for confirmation. Overrides any previous '-f' option(s)")
parsert.add_argument('--watch-the-blood-dripping', default=None, action="store_true", dest="watch_the_blood_dripping",
help=argparse.SUPPRESS)
parsert.add_argument("-R", nargs=1, action=AppendList, dest='rootDirs',
help="Append directories matching the pattern to the vbs_rm search path. Shell-style wildcards ('*?') are supported. This option may be present multiple times to add multiple patterns")
parsert.add_argument('--version', action='version', version=version, help="Print current version and exit succesfully")
parsert.add_argument("pattern", nargs='+',
help="Delete recordings matching these pattern(s). Shell-style wildcards ('*?') are supported.")
# deal with command line
userinput = parsert.parse_args()
# If there is a pattern that is built of only wildcards [i.e. which would match all recordings, i.e. a request to
# delete all recordings ...] we'll ask the user to confirm that this is his/her true intent
if filter(re.compile(r'^((\*[\*\?]*)|([\*\?]*\*)|(\?+\*[\*\?]*))$').match, userinput.pattern) and \
not (userinput.confirmation is dont_ask and userinput.watch_the_blood_dripping is True):
print "###############################################################"
print
print " You've requested to remove ALL FlexBuff/Mark6 recordings"
print
print " For this program to accept this and proceed, you MUST run it"
print " with the following command line flags, to indicate you've "
print " understood the possible consequences:"
print
print " $> vbs_rm -f --watch-the-blood-dripping *"
print
print
print " >>>> You might want to run 'vbs_ls -l' first to <<<<"
print " >>>> get an overview of what you're about to delete <<<<"
print
print
print "###############################################################"
sys.exit(1)
####################################################################################
# Analyze what the user actually wanted
####################################################################################
# 1.) Get the list of mountpoints
mountpoints = reduce(lambda a, pattern: a.update(filter(lambda p: os.path.isdir(p) and os.access(p, os.X_OK|os.R_OK),
glob.glob(pattern))) or a,
userinput.rootDirs or [flexbuff_pattern], set())
# 2.) Zip all mountpoints with all pattern(s) and collect all matching recordings. For each entry split into list of files and list of dirs.
# Filter out recordings that have anything to remove at all and then do that
reduce(remove, filter(maybe_remove,
reduce(index, zip(mountpoints, itertools.repeat(userinput.pattern)), collections.defaultdict(mk(files=list(), dirs=list()))).iteritems()
), userinput.confirmation)