Browse Source

Add jive-toolchain-verify tests

jive-toolchain-verify has two tools: compare-ms-idi.py and
compare-ms-idi-meta.py to compare MS/FITS-IDI files for (meta) data content
and indicate differences.

These tests are now run on the known-good MS/FITS-IDI and the freshly
generated ones to veriy that the (meta) data content remains constant

Log jplotter sequence of commands in logfile

Add new dependencies to README.md

the test suite depends on several external tools, they're now listed, and
documented some more command line options
master
haavee 10 months ago
parent
commit
51c30af749
  1. 60
      ES085A/test_ES085A.py
  2. 16
      README.md
  3. 8
      python/compare_MS_numerically.py

60
ES085A/test_ES085A.py

@ -201,9 +201,10 @@ class TTC(unittest.TestCase):
#
def testAA(self):
rv = None
cmd = "{python} {compare} {gold} es085a_cont.ms".format( python=options.binaries.python,
compare=os.path.join(TTC.__rootdir__, "../python", "compare_MS_numerically.py"),
gold=os.path.join(TTC.__rootdir__, "es085a_cont.ms") )
cmd = "{python} {compare} {gold_ms} es085a_cont.ms".format(
python = options.binaries.python,
compare = os.path.join(TTC.__rootdir__, "../python", "compare_MS_numerically.py"),
gold_ms = os.path.join(TTC.__rootdir__, "es085a_cont.ms") )
# we need to prepare the environment
jplEnv = copy.deepcopy(os.environ)
# Make sure the pythonpath to jplotter is added
@ -256,9 +257,11 @@ class TTC(unittest.TestCase):
def testBA(self):
rv = None
cmd = "{python} {compare} {gold} es085a.ms".format( python=options.binaries.python,
compare=os.path.join(TTC.__rootdir__, "../python", "compare_MS_numerically.py"),
gold=os.path.join(TTC.__rootdir__, "es085a.ms") )
cmd = "{python} {compare} {gold_ms} es085a.ms".format(
python = options.binaries.python,
compare = os.path.join(TTC.__rootdir__, "../python", "compare_MS_numerically.py"),
#gold_ms = os.path.join(TTC.__rootdir__, "es085a-hack.ms") )
gold_ms = os.path.join(TTC.__rootdir__, "es085a.ms") )
# we need to prepare the environment
jplEnv = copy.deepcopy(os.environ)
# Make sure the pythonpath to jplotter is added
@ -315,9 +318,50 @@ class TTC(unittest.TestCase):
# Demand that tConvert ran w/o error
self.assertEqual(rv, 0)
# Perform the n-way diff between gold measurement sets and FITS-IDI files
def EtestE(self):
# Perform the n-way data content diff between gold measurement set and FITS-IDI file
# and those just produced
def testE(self):
rv = None
cmd = "{python} {verify} --ms {gold_ms} --idi {gold_idi} \
--ms es085a_cont.ms --ms es085a.ms --idi ES085A_CONT.IDI --idi ES085A.IDI".format(
python = options.binaries.python,
verify = os.path.join(TTC.__rootdir__, "../jive-toolchain-verify/compare-ms-idi.py"),
gold_ms = os.path.join(TTC.__rootdir__, "es085a_cont.ms"),
gold_idi = os.path.join(TTC.__rootdir__, "ES085A_CONT.IDI"),
)
print("Start", cmd)
with open( os.path.join(TTC.__workdir__, 'testE.log'), 'w' ) as lf:
lf.write( datetime.datetime.now().isoformat() + '\n' )
lf.write( cmd + '\n' )
lf.flush()
process = RUN(SHLEX(cmd), stdout=lf, stderr=lf)
process.wait()
rv = process.returncode
lf.write( "\ncompare-ms-idi.py exited with return code {0}".format(rv) )
TTC.__cleanup__ = TTC.__cleanup__ and (rv==0)
self.assertEqual(rv, 0)
# Perform the n-way meta data content diff between gold measurement set and FITS-IDI file
# and those just produced
def testF(self):
rv = None
cmd = "{python} {verify} --ms {gold_ms} --idi {gold_idi} \
--ms es085a_cont.ms --ms es085a.ms --idi ES085A_CONT.IDI --idi ES085A.IDI".format(
python = options.binaries.python,
verify = os.path.join(TTC.__rootdir__, "../jive-toolchain-verify/compare-ms-idi-meta.py"),
gold_ms = os.path.join(TTC.__rootdir__, "es085a_cont.ms"),
gold_idi = os.path.join(TTC.__rootdir__, "ES085A_CONT.IDI"),
)
print("Start", cmd)
with open( os.path.join(TTC.__workdir__, 'testF.log'), 'w' ) as lf:
lf.write( datetime.datetime.now().isoformat() + '\n' )
lf.write( cmd + '\n' )
lf.flush()
process = RUN(SHLEX(cmd), stdout=lf, stderr=lf)
process.wait()
rv = process.returncode
lf.write( "\ncompare-ms-idi-meta.py exited with return code {0}".format(rv) )
TTC.__cleanup__ = TTC.__cleanup__ and (rv==0)
self.assertEqual(rv, 0)
# Leave this one as last test case to trigger potential cleanup

16
README.md

@ -1,6 +1,6 @@
# jive-toolchain-regression
Provide regression test(s) for j2ms2 / tConvert that can be run in (semi)automated fashion.
Provide regression test(s) for [j2ms2 / tConvert](https://code.jive.eu/verkout/jive-casa) that can be run in (semi)automated fashion.
The programs translate JIVE software correlator (SFXC) output into CASA MeasurementSet and FITS-IDI format.
The goal of this testsuite is to verify that changes in the source code do not alter the data and meta data content unexpectedly by running the toolchain-under-test on data with known properties.
@ -14,6 +14,14 @@ The test suite tests the following:
The test cleans up after itself if all tests have passed without errors.
Otherwise all artefacts (produced data sets and log files) will be left and a message is printed containing the location of the artefacts.
### Dependencies
The regression test suite depends on the following other tools:
- `python` (2 or 3) w/ modules:
- `unittest`
- [`python-casacore`](https://github.com/casacore/python-casacore)
- [`jplotter`](https://github.com/haavee/jiveplot) (this tool also requires `python-casacore`)
- `git` - to automatically download the [`jive-toolchain-verify`](https://code.jive.eu/verkout/jive-toolchain-verify) MeasurementSet/FITS-IDI comparison tools
# Running
@ -26,6 +34,12 @@ $> ./main [options]
You can run `./main -h` for help on `[options]`
## Which python and/or jplotter to use?
Pass any/all of the following options to `main` to set the path to the desired version to use:
--python PATH Path to the Python interpreter to use
--jplotter PATH Path to the jplotter program to use
## Which j2ms2/tConvert to test?

8
python/compare_MS_numerically.py

@ -62,8 +62,12 @@ try:
oldexit, oldhook = sys.exit, sys.excepthook
sys.exit = myexitfn
sys.excepthook = myexcepthook
jplotter.run_plotter( command.readstring(mk_plots(tempfile=os.path.join(tmpdir,'plotfile.ps'),
reference=sys.argv[1], test=sys.argv[2])),
cmd = mk_plots(tempfile=os.path.join(tmpdir,'plotfile.ps'),
reference=sys.argv[1], test=sys.argv[2])
print("====> jplotter script")
print(cmd)
print("<==== jplotter script")
jplotter.run_plotter( command.readstring(cmd),
debug=True, excepthook=myexcepthook )
sys.excepthook = oldhook
sys.exit = oldexit

Loading…
Cancel
Save