Browse Source

Can now run numerical equivalence test!

Added command line option
    --jplotter /path/to/jplotter (default: taken from environment)

and wrote a jplotter postprocessing plugin that does sys.exit(-1) if the
accumulated difference between the last to plots > tolerance (currently: 1e-7)

The jplotter is driven from a toplevel python script that:
    - tests if both measurementsets to compare are measurement sets
      (jplotter is too forgiving upon that kind of errors)
    - plays nasty trick with sys.excepthook and sys.exit functions:
      replaces both before the call to jplotter.run_plotter(...)
      to catch execptions and sys.exit(!=0) and translate into
      error being set.
    - exits with 0 if the two measurement sets compare numerically equal as
      defined by the compare_ms_data.py:compare_ms_data() function

The reasons for that last bit are twofold:
    - a temporary dir+plotfile are created to do the batchplotting into
      which require cleanup in all cases of exit (succes or #fail)
    - we must be able to propagate the succes#fail state of the comparison
      to our caller
master
haavee 10 months ago
parent
commit
d7e8b96b43
  1. 143
      ES085A/test_ES085A.py
  2. 79
      python/compare_MS_numerically.py
  3. 79
      python/compare_ms_data.py

143
ES085A/test_ES085A.py

@ -1,5 +1,5 @@
from __future__ import print_function
import os, sys, glob, shutil, shlex, argparse, tempfile, subprocess, datetime, unittest
import os, sys, copy, glob, shutil, shlex, argparse, tempfile, operator, subprocess, datetime, unittest
# j2ms2 + tConvert come as a pair.
# We need to be able to specify where to take the binaries from.
@ -8,6 +8,7 @@ import os, sys, glob, shutil, shlex, argparse, tempfile, subprocess, datetime, u
# path/to/build_dir/apps/[j2ms2|tConvert]/\\1
# However, when installed, they're accessible as:
# path/to/install_dir/bin/{j2ms2,tConvert}
sys.executable
# from python shutil source code (https://github.com/python/cpython/blob/master/Lib/shutil.py)
def _access_check(fn, mode):
@ -26,6 +27,20 @@ def which(cmd):
class EnvironmentBinaries(object):
_j2ms2_ = which('j2ms2')
_tConvert_ = which('tConvert')
_jplotter_ = which('jplotter')
def get_j2ms2(self):
return self._j2ms2_fn_(self)
def get_tConvert(self):
return self._tConvert_fn_(self)
def get_jplotter(self):
return self._jplotter_fn_(self)
def get_python(self):
return self._python_fn_(self)
j2ms2 = property(get_j2ms2)
tConvert = property(get_tConvert)
jplotter = property(get_jplotter)
python = property(get_python)
# path argument baseclass for Argparse;
# will set "__path" to value of the option on the command line
@ -33,26 +48,41 @@ class BaseAction(argparse.Action):
def __init__(self, option_strings, dest, nargs=None, **kwargs):
if nargs is not None:
raise ValueError("nargs not allowed")
# extract the variable to update in the destination
self.p__ = kwargs.pop('path', None)
if self.p__ is None:
raise RuntimeError("This action requires a path= keyword argument")
super(BaseAction, self).__init__(option_strings, dest, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
self.path__ = values
setattr(namespace, self.dest, self)
# If the namespace does not already have an attribute called destination
# then we install ourselves as one
if not hasattr(namespace, self.dest):
setattr(namespace, self.dest, self)
# Now update the object in the target namespace
setattr(getattr(namespace, self.dest), self.p__, values)
# shorthand to set attributes in o, with name taken from d.keys and to values set by d.values
def updateObj(o, d):
for (nm, fn) in d.items():
setattr(o, nm, fn)
# Modify base class to extract / return paths for j2ms2/tConvert:
# You provide the functions to generate the path to the binary/ies
# via keywword args: SetBinaries(...., j2ms2 = lambda s: ..., tConvert=lambda s: ...)
def SetBinaries(cls, **kwargs):
fnsToSet = copy.deepcopy(kwargs)
class NewClass(cls):
def __init__(self, *a, **k):
super(cls, self).__init__(*a, **k)
self._j2ms2_fn_, self. _tConvert_fn_ = (kwargs.get('j2ms2'), kwargs.get('tConvert'))
def get_j2ms2(self):
return self._j2ms2_fn_(self)
def get_tConvert(self):
return self._tConvert_fn_(self)
j2ms2 = property(get_j2ms2)
tConvert = property(get_tConvert)
updateObj(self, fnsToSet)
super(NewClass, self).__init__(*a, **k)
def __call__(self, parser, namespace, values, option_string=None):
# when the action is called we need to update stuff
if not hasattr(namespace, self.dest):
setattr(namespace, self.dest, self)
updateObj(getattr(namespace, self.dest), fnsToSet)#self.fnsToSet)
return super(NewClass, self).__call__(parser, namespace, values, option_string)
return NewClass
def Raise(err):
@ -61,30 +91,36 @@ def Raise(err):
# Create classes that can be given to Argparse as action and default
# into immediately useful objects
DefaultBinaries = SetBinaries(EnvironmentBinaries,
j2ms2=lambda _: EnvironmentBinaries._j2ms2_ or Raise("No j2ms2 found in your $PATH"),
tConvert=lambda _: EnvironmentBinaries._tConvert_ or Raise("No tConvert found in your $PATH"))
InstallDirAction = SetBinaries(BaseAction, j2ms2=lambda s: os.path.join(s.path__, "bin/j2ms2"),
tConvert=lambda s: os.path.join(s.path__, "bin/tConvert"))
BuildDirAction = SetBinaries(BaseAction, j2ms2=lambda s: os.path.join(s.path__, "apps/j2ms2/j2ms2"),
tConvert=lambda s: os.path.join(s.path__, "apps/tConvert/tConvert"))
_j2ms2_fn_=lambda _: EnvironmentBinaries._j2ms2_ or Raise("No j2ms2 found in your $PATH"),
_tConvert_fn_=lambda _: EnvironmentBinaries._tConvert_ or Raise("No tConvert found in your $PATH"),
_jplotter_fn_=lambda _: EnvironmentBinaries._jplotter_ or Raise("No jplotter found in your $PATH"),
_python_fn_=lambda _: sys.executable)
# The InstallDir- and BuildDirAction will set the 'path__' attribute if on the command line
InstallDirAction = SetBinaries(BaseAction, _j2ms2_fn_=lambda s: os.path.join(s.path__, "bin/j2ms2"),
_tConvert_fn_=lambda s: os.path.join(s.path__, "bin/tConvert"))
BuildDirAction = SetBinaries(BaseAction, _j2ms2_fn_=lambda s: os.path.join(s.path__, "apps/j2ms2/j2ms2"),
_tConvert_fn_=lambda s: os.path.join(s.path__, "apps/tConvert/tConvert"))
# The jplotterAction will set the 'jplpath__' attribute if on the command line
jplotterAction = SetBinaries(BaseAction, _jplotter_fn_=lambda s: os.path.join(s.jplpath__, "jplotter"))
# Following https://stackoverflow.com/a/17259773
# Strip out options specifically meant for us and replace sys.argv[1:] with what remains
binarys = DefaultBinaries()
parsert = argparse.ArgumentParser(add_help=True)
mutexgr = parsert.add_mutually_exclusive_group()
mutexgr.add_argument('--cmake-install-dir', help="test j2ms2/tConvert binaries from install_dir/bin",
dest='binaries', action=InstallDirAction, default=DefaultBinaries())
dest='binaries', action=InstallDirAction, path='path__', default=binarys)#DefaultBinaries())
mutexgr.add_argument('--cmake-build-dir', help="test j2ms2/tConvert binaries from build_dir/app/{j2ms2,tConvert}/",
dest='binaries', action=BuildDirAction, default=DefaultBinaries())
dest='binaries', action=BuildDirAction, path='path__', default=binarys)#DefaultBinaries())
parsert.add_argument('--jplotter', help="Path to jplotter to use (default: from $PATH)",
dest='binaries', action=jplotterAction, path='jplpath__', default=binarys)#DefaultBinaries())
options, args = parsert.parse_known_args()
sys.argv[1:] = args
## Yes, technically tests should be able to run in any order
## but we're testing a toolchain here where e.g. step N+1
## uses a product generated in step N
@ -159,6 +195,35 @@ class TTC(unittest.TestCase):
# Demand that j2ms2 ran w/o error
self.assertEqual(rv, 0)
#
# Compare the generated lisfile MS with the known-good one
# We use the jplotter helper for this
#
def testAA(self):
rv = None
cmd = "{python} {compare} {gold} es085a_cont.ms".format( python=options.binaries.python,
compare=os.path.join(TTC.__rootdir__, "../python", "compare_MS_numerically.py"),
gold=os.path.join(TTC.__rootdir__, "es085a_cont.ms") )
# we need to prepare the environment
jplEnv = copy.deepcopy(os.environ)
# Make sure the pythonpath to jplotter is added
jplEnv['PYTHONPATH'] = ":".join(filter(operator.truth,
[os.path.join(TTC.__rootdir__, "../python"),
os.path.dirname(options.binaries.jplotter),
jplEnv.get('PYTHONPATH', None)]))
print("Start", cmd)
with open( os.path.join(TTC.__workdir__, 'testAA.log'), 'w' ) as lf:
lf.write( datetime.datetime.now().isoformat() + '\n' )
lf.write( cmd + '\n' )
lf.flush()
process = RUN(SHLEX(cmd), stdout=lf, stderr=lf, env=jplEnv)
process.wait()
rv = process.returncode
lf.write( "\ncomparison exited with return code {0}".format(rv) )
TTC.__cleanup__ = TTC.__cleanup__ and (rv==0)
# Demand that the comparison ran w/o error
self.assertEqual(rv, 0)
# Attempt to run j2ms2 in "manual" mode, i.e. specifiying cor files & output MS file on the command line
# j2ms2 -o <ms> 24427/*.cor
# Note: we can do the globbing on "24427/*.cor" since the setup code has done a
@ -180,6 +245,32 @@ class TTC(unittest.TestCase):
# Demand that j2ms2 ran w/o error
self.assertEqual(rv, 0)
def testBA(self):
rv = None
cmd = "{python} {compare} {gold} es085a.ms".format( python=options.binaries.python,
compare=os.path.join(TTC.__rootdir__, "../python", "compare_MS_numerically.py"),
gold=os.path.join(TTC.__rootdir__, "es085a.ms") )
# we need to prepare the environment
jplEnv = copy.deepcopy(os.environ)
# Make sure the pythonpath to jplotter is added
jplEnv['PYTHONPATH'] = ":".join(filter(operator.truth,
[os.path.join(TTC.__rootdir__, "../python"),
os.path.dirname(options.binaries.jplotter),
jplEnv.get('PYTHONPATH', None)]))
print("Start", cmd)
with open( os.path.join(TTC.__workdir__, 'testBA.log'), 'w' ) as lf:
lf.write( datetime.datetime.now().isoformat() + '\n' )
lf.write( cmd + '\n' )
lf.flush()
process = RUN(SHLEX(cmd), stdout=lf, stderr=lf, env=jplEnv)
process.wait()
rv = process.returncode
lf.write( "\ncomparison exited with return code {0}".format(rv) )
TTC.__cleanup__ = TTC.__cleanup__ and (rv==0)
# Demand that the comparison ran w/o error
self.assertEqual(rv, 0)
# Run tConvert on the lis-file MS
def testC(self):
rv = None
@ -197,6 +288,7 @@ class TTC(unittest.TestCase):
# Demand that tConvert ran w/o error
self.assertEqual(rv, 0)
# Run tConvert on the manual MS
def testD(self):
rv = None
@ -214,6 +306,11 @@ class TTC(unittest.TestCase):
# Demand that tConvert ran w/o error
self.assertEqual(rv, 0)
# Perform the n-way diff between gold measurement sets and FITS-IDI files
def EtestE(self):
rv = None
self.assertEqual(rv, 0)
# Leave this one as last test case to trigger potential cleanup
# in the associated tearDown() call
def testZ(self):

79
python/compare_MS_numerically.py

@ -0,0 +1,79 @@
# Run the jplotter to produce the same plot from two measurements and expects them to be be totally equivalent.
#
# "Plot" is exaggerated - the code does not compare the produced PNGs or JPGs but
# installs a jiveplot "postprocessing" hook that is called before generating
# the images. This hook has access to the raw data to be plotted and can modify it,
# or, in this case, compare it for differences to a previously stored version
# of the raw data.
#
from __future__ import print_function
import __main__, os, sys, command, jplotter, shutil, tempfile, traceback, pyrap
if len(sys.argv)!=3:
raise RuntimeError("Usage: {0} path/to/reference.ms path/to/test.ms".format( sys.argv[0] ))
# Since jplotter is very forgiving when the user attempts to open a non-existing MS
# it doesn't generate an error in that case. So we verify here that both measurement sets at least _seem_ to exist
with pyrap.tables.table(sys.argv[1]) as test:
pass
with pyrap.tables.table(sys.argv[2]) as test:
pass
# add the path to this module such that Python can load the compare_ms_data postprocessing module
sys.path.insert(0, os.path.dirname(__main__.__file__))
# This is the sequence of commands what we let jiveplot execute
mk_plots = """
refile {tempfile}
postprocess ./compare_ms_data.compare_ms_data
ms {reference}
indexr; scan 18 19 33; pt anpfreq; avt sum; solint none; y0 local; y1 local; multi t; new src t; sort src bl; pl;
ms {test}
indexr; scan 18 19 33; pt anpfreq; avt sum; solint none; y0 local; y1 local; multi t; new src t; sort src bl; pl;
""".format
# We need to capture errors without exiting Python
# because we need to do cleanup of the /tmp/ directory (the plots)
error = False
def mkerrf(pfx):
global error
def actualerrf(msg):
print("MYMKERRF:", pfx, msg)
error = True
return actualerrf
jplotter.hvutil.mkerrf = mkerrf
def myexcepthook(tp, v, tb):
global error
error = True
print("MYEXCEPTHOOK:", tp, ":", v, "\n")
traceback.print_tb(tb)
def myexitfn(exitval):
global error
error = (exitval != 0)
print("MYEXITFN: exitcode=", exitval)
# Need a temp file name to send output to
tmpdir = tempfile.mkdtemp()
# make sure we don't let any exception escape - we need to do cleanup
try:
oldexit, oldhook = sys.exit, sys.excepthook
sys.exit = myexitfn
sys.excepthook = myexcepthook
jplotter.run_plotter( command.readstring(mk_plots(tempfile=os.path.join(tmpdir,'plotfile.ps'),
reference=sys.argv[1], test=sys.argv[2])),
debug=True, excepthook=myexcepthook )
sys.excepthook = oldhook
sys.exit = oldexit
except Exception as e:
print("TF?", e)
error = True
pass
# clean up after ourselves
shutil.rmtree( tmpdir )
# did the test finish succesfully or not?
sys.exit( -1 if error else 0 )

79
python/compare_ms_data.py

@ -0,0 +1,79 @@
from __future__ import print_function
import sys, numpy, plots, math, operator, copy
COPY = copy.deepcopy
# define a post-processing operation that always remembers the last data set and
# computes statistical differences between that one and the next data set.
# After that, the new data set becomes the last one.
# Purpose: to verify that two chunks of data extraced from measurement sets that *should*
# be equal, are, in fact, equal.
# If a new type of data set is detected (new plot type) the last data set is erased,
# this data set becomes the stored one, and no output is generated
last_dataset = None
tolerance = 1e-7
def plotar2unidict(plotar):
rv = plots.Dict()
rv.plotType = plotar.plotType
# loop over all plots and datasets-within-plot
for k in plotar.keys():
for d in plotar[k].keys():
# get the full data set label - we have access to all the data set's properties (FQ, SB, POL etc)
n = plots.join_label(k, d)
# and make a copy of the dataset
rv[n] = COPY( plotar[k][d] )
return rv
# this is the function to pass to `postprocess ...`
def compare_ms_data(plotar, ms2mappings):
global last_dataset
# Whatever we need to do - this can be done unconditionally
new_dataset = plotar2unidict( plotar )
error = ""
# Check if we need to do anything at all
if last_dataset is not None and last_dataset.plotType == new_dataset.plotType:
# OK check all common keys
old_keys = set(last_dataset.keys())
new_keys = set(new_dataset.keys())
# if the sets are not equal, the data will also not compare equal!
if old_keys == new_keys:
# inspect x and y separately, add up all the diffs
dx, dy = 0, 0
for k in old_keys:
ods = last_dataset[ k ]
nds = new_dataset[ k ]
dx = numpy.add(numpy.abs( ods.xval - nds.xval ), dx)
dy = numpy.add(numpy.abs( ods.yval - nds.yval ), dy)
#print("dx=", dx)
#print("dy=", dy)
if numpy.any( dx>abs(tolerance) ):
print(">>> compare_data: total diffs in x exceed tolerance")
print(" tolerance=", tolerance)
print(" accum. dx=", dx)
error += "Datasets mismatch in X according to tolerance. "
if numpy.any( dy>abs(tolerance) ):
print(">>> compare_data: total diffs in y exceed tolerance")
print(" tolerance=", tolerance)
print(" accum. dy=", dy)
error = "Datasets mismatch in Y according to tolerance. "
else:
print(">>> compare_data: The datasets to compare have different data content?!")
common = old_keys & new_keys
only_o = old_keys - common
only_n = new_keys - common
print(" Common keys:", len(common))
print(" Uniq in Old:", len(only_o))
print(" Uniq in New:", len(only_n))
error += "Datasets mismatch in content. "
#with open('/tmp/oldkeys.txt', 'w') as f:
# list( map(lambda s: f.write(str(s) + '\n'), sorted(only_o)) )
#with open('/tmp/newkeys.txt', 'w') as f:
# list( map(lambda s: f.write(str(s) + '\n'), sorted(only_n)) )
# Install new dataset as new last dataset
last_dataset = new_dataset
if error:
sys.exit( -1 )
raise RuntimeError(error)
Loading…
Cancel
Save