Browse Source

Support for (no)squash_{antenna|source}_table

By popular demand all combinations should be possible now
pull/3/head
haavee 10 months ago
parent
commit
9b92dd3dfd
  1. 10
      apps/j2ms2/j2ms2.cc
  2. 173
      jive/j2ms/JIVEMSFiller.cc
  3. 8
      jive/j2ms/JIVEMSFiller.h

10
apps/j2ms2/j2ms2.cc

@ -403,10 +403,16 @@ void Usage( void ) {
" default is to delete if station has no scans;" << endl <<
" data_good honour the 'data good' property from the VEX" << endl <<
" file (will be added to nominal scan start);" << endl <<
" - fo:squash_antenna_table" << endl <<
" - fo:(no)squash_antenna_table" << endl <<
" only write entries in the antenna table for stations" << endl <<
" that actually have data in the main table." << endl <<
" that actually have data in the main table" << endl <<
" or write all ALL antennas in the ANTENNA table." << endl <<
" Default: write ALL antennas into the ANTENNA subtable." << endl <<
" - fo:(no)squash_source_table" << endl <<
" only write entries in the FIELD table for sources" << endl <<
" that actually have data in the main table" << endl <<
" or write all ALL sources in the FIELD table." << endl <<
" Default: only write sources with data into the FIELD subtable." << endl <<
" <buffer> visibility buffer identifier, typically:" << endl <<
" for SFXC: '<job>/<scan>.cor'" << endl <<
" for UniBoard: '<job>.json'" << endl <<

173
jive/j2ms/JIVEMSFiller.cc

@ -149,7 +149,8 @@ Vector<Double> operator*( const RotMatrix& lhside, const Vector<Double>& rhside
JIVEMSFiller::JIVEMSFiller( const String& opt ):
myOptions( opt ),
updateAntennaAndFeedTable( &JIVEMSFiller::updateAntennaAndFeedTable_once )
updateAntennaAndFeedTable( &JIVEMSFiller::updateAntennaAndFeedTable_once ),
updateFieldTable( &JIVEMSFiller::updateFieldTable_dynamic )
{
// cout << "Created JIVEMSFiller with opt=" << myOptions << endl;
//
@ -194,6 +195,12 @@ JIVEMSFiller::JIVEMSFiller( const String& opt ):
fspec = tmp(s, l);
} else if( tmp=="squash_antenna_table" ) {
updateAntennaAndFeedTable = &JIVEMSFiller::updateAntennaAndFeedTable_dynamic;
} else if( tmp=="nosquash_antenna_table" ) {
updateAntennaAndFeedTable = &JIVEMSFiller::updateAntennaAndFeedTable_once;
} else if( tmp=="squash_source_table" ) {
updateFieldTable = &JIVEMSFiller::updateFieldTable_dynamic;
} else if( tmp=="nosquash_source_table" ) {
updateFieldTable = &JIVEMSFiller::updateFieldTable_once;
} else if( curopt->size() ) {
cerr << "JIVEMSFiller: unsupported option `" << *curopt << "'" << endl;
}
@ -869,7 +876,7 @@ Bool JIVEMSFiller::fillMSfromVisibilityBuffer( MeasurementSet& msref,
<< " Corr: " << setup << endl;
// Look up the source of this scan in the experiment's source-list
if( (currsourceindex=JIVEMSFiller::updateFieldTable(msref, *curScanptr))<0 )
if( (currsourceindex=(this->*updateFieldTable)(msref, *curScanptr))<0 )
THROW_JEXCEPT(String("Source '")+curScanptr->getSourceCode()+"' not found in database?! Cannot go on!");
// Try to get the frequencysetup for this scan....
@ -1696,7 +1703,7 @@ void JIVEMSFiller::rebuild_caches(MeasurementSet const& ms) {
nSp, (sideband == 0 ? chans[nSp-1] : chans[0]),
spwcols.totalBandwidth()(spwid), sideband) );
}
cout << "Loaded " << spectral_window_table.size() << " entries from SPECTRAL_WINDOW" << endl;
cout << "Loaded " << spectral_window_table.size() << " entries from " << spwcols.nrow() << " SPECTRAL_WINDOWs" << endl;
// Deal with the POLARIZATIONs
ROMSPolarizationColumns polcols( ms.polarization() );
@ -1757,7 +1764,7 @@ void JIVEMSFiller::rebuild_caches(MeasurementSet const& ms) {
// Olreit!
polarization_table.push_back( polarization_entry_type(polid, j2, l2p) );
}
cout << "Loaded " << polarization_table.size() << " entries from POLARIZATION" << endl;
cout << "Loaded " << polarization_table.size() << " entries from " << polcols.nrow() << " POLARIZATIONs" << endl;
// Load DATA_DESCRIPTION table
ROMSDataDescColumns ddcols( ms.dataDescription() );
@ -1769,7 +1776,7 @@ void JIVEMSFiller::rebuild_caches(MeasurementSet const& ms) {
continue;
datadescription_table.push_back(datadescription_entry_type((Int)ddid, ddcols.spectralWindowId()(ddid), ddcols.polarizationId()(ddid)));
}
cout << "Loaded " << datadescription_table.size() << " entries from DATA_DESCRIPTION" << endl;
cout << "Loaded " << datadescription_table.size() << " entries from " << ddcols.nrow() << " DATA_DESCRIPTIONs" << endl;
// Load ANTENNA table ...
ROMSAntennaColumns antcols( ms.antenna() );
@ -1781,7 +1788,7 @@ void JIVEMSFiller::rebuild_caches(MeasurementSet const& ms) {
antenna_table.push_back(antenna_entry_type(antid, antcols.name()(antid), antcols.station()(antid), antcols.mount()(antid),
antcols.dishDiameter()(antid), antcols.position()(antid), antcols.offset()(antid)));
}
cout << "Loaded " << antenna_table.size() << " entries from ANTENNA" << endl;
cout << "Loaded " << antenna_table.size() << " entries from " << antcols.nrow() << " ANTENNAs" << endl;
// Load FEED table
ROMSFeedColumns feedcols( ms.feed() );
@ -1829,7 +1836,7 @@ void JIVEMSFiller::rebuild_caches(MeasurementSet const& ms) {
field_table.push_back(field_entry_type((Int)field, fieldcols.sourceId().get(field), fieldcols.name().get(field),
fieldcols.time().get(field), delayDir, phaseDir, referenceDir));
}
cout << "Loaded " << field_table.size() << " entries from FIELD" << endl;
cout << "Loaded " << field_table.size() << " entries from " << fieldcols.nrow() << " FIELDs" << endl;
// Load the STATE table
ROMSStateColumns statecols( ms.state() );
@ -1842,7 +1849,7 @@ void JIVEMSFiller::rebuild_caches(MeasurementSet const& ms) {
statecols.cal().get(state), statecols.load().get(state),
statecols.subScan().get(state), statecols.obsMode().get(state)) );
}
cout << "Loaded " << state_table.size() << " entries from STATE" << endl;
cout << "Loaded " << state_table.size() << " entries from " << statecols.nrow() << " STATEs" << endl;
}
@ -2027,10 +2034,12 @@ void writeAntennaTable(MeasurementSet& msref, VisibilityBuffer const& visbuf) {
AntennaDB const& antdb( expref.getAntennaDB() );
FrequencyDB const& freqdb( expref.getFrequencyDB() );
std::cout << "Writing AntennaAndFeed ONCE" << std::endl;
std::cout << "Writing AntennaAndFeed ONCE" << std::endl;
if( freqdb.nEntries()==0 )
if( freqdb.nEntries()==0 ) {
std::cerr << "Experiment's frequency table is empty. Giving up." << std::endl;
exit(-1);
}
// If we add an antenna, we will immediately write a corresponding entry
// in the FEED table.
@ -2132,6 +2141,81 @@ std::cout << "Writing AntennaAndFeed ONCE" << std::endl;
return;
}
// Write all entries from the associated Experiment's SourceDB 1:1 to the
// MeasurementSet FIELD subtable and add the sources to the in-memory
// field_table as well.
void writeFieldTable(MeasurementSet& msref, VisibilityBuffer const& visbuf) {
SourceInfo si;
MSField& fieldref( msref.field() );
MSFieldColumns fieldcols( fieldref );
Experiment const& expref( visbuf.getExperiment() );
SourceDB const& srcdb( expref.getSourceDB() );
std::cout << "Writing Field table ONCE" << std::endl;
if( srcdb.nrSources()==0 ) {
std::cerr << "Experiment's source table is empty. Giving up." << std::endl;
exit(-1);
}
// ====> Start adding fields
for( uInt id = 0; id<srcdb.nrSources(); id++ ) {
if( !srcdb.searchSource(si, (casacore::Int)id) )
THROW_JEXCEPT(String("Request for experiment source #") << id << "' fails, no such source?!");
if( si.getId()!=(Int)id )
THROW_JEXCEPT("Source #" << id << " does not report same id in getId()");
// Do some pre-processing on the listed source direction in order to be
// able to look for a correctly matching entry.
// MDirection should yield vector of length 2: RA, DEC
Vector<Double> const srcdir( (si.getDirection().getValue()).get() );
if( srcdir.size()!=2 )
THROW_JEXCEPT("The source '" << si.getCode() << "'s direction is not just RA, DEC?!");
// In the Field table it is allowed to have a direction as a polynomial
// series in time with NUM_POLY coefficients, written as a matrix of
// 2xNUM_POLY (Ra, Dec) * [num_poly + 1] coefficients.
// casacore Matrix<T> is (nrow, ncolumn) so the direction matrices look:
// RA , RA(t)_0 , RA(t)_1 , ... [expansion coefficients of RA(t) ]
// DEC, DEC(t)_0, DEC(t)_2, ... [expansion coefficients of DEC(t)]
//
// Our code assumes the sources have constant position, thus NUM_POLY==0
// but we must list NUM_POLY+1 "coefficients", including the 0th order
// (constant term)
//
// Let's see if we can go from Vector => properly shaped Matrix in one
// fell sweep
fielddir_type const fielddir( srcdir.reform( IPosition(2, 2/*ra, dec*/, 1/*only 1 term in the polynomials*/) ) );
// Now we can build the field table entry we're looking for.
// Note that our code always looks for [and inserts if not found] an entry with:
// * time = 0.0 [must be valid for all times]
// * delay_dir == phase_dir == reference_dir == source position from SourceDB
// * source_id == -1 [no associated entry in the source table]
field_entry_type const field(si.getId(), -1 /*No associated source*/,
si.getCode(), casacore::Double(0.0) /*Valid for all times*/,
fielddir, fielddir, fielddir /*All three directions equal*/);
// Now add to the source table (in memory) ...
field_table.push_back( field );
// ... and to the MSField subtable
fieldref.addRow();
fieldcols.name().put( id, field.__m_FieldName );
fieldcols.sourceId().put( id, field.__m_SourceId );
fieldcols.time().put( id, field.__m_Time );
fieldcols.numPoly().put( id, 0 );
fieldcols.delayDir().put( id, field.__m_DelayDir );
fieldcols.phaseDir().put( id, field.__m_PhaseDir );
fieldcols.referenceDir().put( id, field.__m_ReferenceDir );
fieldcols.flagRow().put( id, False );
}
fieldref.flush();
return;
}
// UPDATES the ANTENNA and FEED tables for the given experiment into the
// indicated MS.
//
@ -2207,8 +2291,10 @@ antennafeed_pair_type JIVEMSFiller::updateAntennaAndFeedTable_once(MeasurementSe
//j2ms::log2polmap const l2p( scanref.getVisibilityBuffer().correlatorSetup().l2pmap() );
// Step 1. Find the antenna in the experiment's database
if( !antdb.searchAntenna(ai, (const Int)ant) )
if( !antdb.searchAntenna(ai, (const Int)ant) ) {
std::cerr << "Antenna #" << ant << " not found in experiment's AntennaDB. Giving up." << std::endl;
exit(-1);
}
// If the antenna table of the MS is empty write it all at once
if( msref.antenna().nrow()==0 )
@ -2220,8 +2306,10 @@ antennafeed_pair_type JIVEMSFiller::updateAntennaAndFeedTable_once(MeasurementSe
antenna_table_type::const_iterator curANT = std::find(antenna_table.begin(), antenna_table.end(), antenna);
// No such antenna??? Then you're fucked.
if( curANT==antenna_table.end() )
if( curANT==antenna_table.end() ) {
std::cerr << "Antenna #" << ant << " not found in the experiment's antenna table. Giving up." << std::endl;
exit(-1);
}
// With this fucking limitation (Aug/2018) we cannot support anything different
// from one feed. For your sake I hope it's the right one. We can't
@ -2260,7 +2348,7 @@ antennafeed_pair_type JIVEMSFiller::updateAntennaAndFeedTable_dynamic(Measuremen
MSAntennaColumns antabcols( antab );
casacore::uInt const newAntId( antab.nrow() );
std::cout << "Update ANTENNA/FEED dynamic, add antenna " << antenna.__m_AntennaName << std::endl;
std::cout << "Update ANTENNA/FEED dynamic, add antenna " << antenna.__m_AntennaName << std::endl;
antab.addRow();
antabcols.name().put( newAntId, antenna.__m_AntennaName);
antabcols.dishDiameter().put( newAntId, antenna.__m_Diameter );
@ -2790,13 +2878,13 @@ const Vector<Double>& JIVEMSFiller::calcUVW( const Visibility& vref ) {
// VEX file into the measurement sets because now only the
// fields whose data is actually written into the MS are
// exported into the field table
casacore::Int JIVEMSFiller::updateFieldTable( MeasurementSet& msref, const j2ms::Scan& scanref ) {
// 29-01-2021 Need to support both dynamic + write all at once modes
casacore::Int JIVEMSFiller::updateFieldTable_dynamic( MeasurementSet& msref, const j2ms::Scan& scanref ) {
SourceInfo si;
Experiment const& expref( scanref.getExperiment() );
SourceDB const& srcdb( expref.getSourceDB() );
casacore::String const srccode( scanref.getSourceCode() );
// Step 1. Find the source's information
if( !srcdb.searchSource(si, srccode) )
THROW_JEXCEPT(String("Request for experiment source '")+srccode+"' fails, no such source?!");
@ -2861,6 +2949,61 @@ casacore::Int JIVEMSFiller::updateFieldTable( MeasurementSet& msref, const j2ms
return curFIELD->__m_FieldId;
}
casacore::Int JIVEMSFiller::updateFieldTable_once( MeasurementSet& msref, const j2ms::Scan& scanref ) {
SourceInfo si;
Experiment const& expref( scanref.getExperiment() );
SourceDB const& srcdb( expref.getSourceDB() );
casacore::String const srccode( scanref.getSourceCode() );
// Step 1. Find the source's information
if( !srcdb.searchSource(si, srccode) ) {
std::cerr << "Source '"+srccode+"' not found in experiment's SourceDB. Giving up." << std::endl;
exit(-1);
}
// If the field table of the MS is empty write it all at once
if( msref.field().nrow()==0 )
writeFieldTable(msref, scanref.getVisibilityBuffer());
// Do some pre-processing on the listed source direction in order to be
// able to look for a correctly matching entry.
// MDirection should yield vector of length 2: RA, DEC
Vector<Double> const srcdir( (si.getDirection().getValue()).get() );
if( srcdir.size()!=2 )
THROW_JEXCEPT(String("The source '")+srccode+"' direction is not just RA, DEC?!");
// In the Field table it is allowed to have a direction as a polynomial
// series in time with NUM_POLY coefficients, written as a matrix of
// 2xNUM_POLY (Ra, Dec) * [num_poly + 1] coefficients.
// casacore Matrix<T> is (nrow, ncolumn) so the direction matrices look:
// RA , RA(t)_0 , RA(t)_1 , ... [expansion coefficients of RA(t) ]
// DEC, DEC(t)_0, DEC(t)_2, ... [expansion coefficients of DEC(t)]
//
// Our code assumes the sources have constant position, thus NUM_POLY==0
// but we must list NUM_POLY+1 "coefficients", including the 0th order
// (constant term)
//
// Let's see if we can go from Vector => properly shaped Matrix in one
// fell sweep
fielddir_type const fielddir( srcdir.reform( IPosition(2, 2/*ra, dec*/, 1/*only 1 term in the polynomials*/) ) );
// Now we can build the field table entry we're looking for.
// Note that our code always looks for [and inserts if not found] an entry with:
// * time = 0.0 [must be valid for all times]
// * delay_dir == phase_dir == reference_dir == source position from SourceDB
// * source_id == -1 [no associated entry in the source table]
field_entry_type const field(-1 /*No field id yet!*/, -1 /*No associated source*/,
srccode, casacore::Double(0.0) /*Valid for all times*/,
fielddir, fielddir, fielddir /*All three directions equal*/);
field_table_type::const_iterator curFIELD = std::find(field_table.begin(), field_table.end(), field);
// Ah?
if( curFIELD==field_table.end() ) {
std::cerr << "Source '" << srccode+"' not found in the experiment's field table. Giving up." << std::endl;
exit(-1);
}
return curFIELD->__m_FieldId;
}
// Update the processor table with a new pass_id
//
// * NO CHECKING FOR DUPLICATES IS DONE HERE *

8
jive/j2ms/JIVEMSFiller.h

@ -127,7 +127,13 @@ private:
// Return the FIELD_ID of the source (field) of the indicated scan.
// Matches a row in the field table or adds a new row if no such row
// was found
casacore::Int updateFieldTable(casacore::MeasurementSet& msref, j2ms::Scan const& scanref);
// HV: 29 Jan 2021: Need to support (no)squash field table too,
// follow same approach as antenna, see above
typedef casacore::Int (JIVEMSFiller::*updateField_fn)(casacore::MeasurementSet&, j2ms::Scan const&);
updateField_fn updateFieldTable;
casacore::Int updateFieldTable_once(casacore::MeasurementSet& msref, j2ms::Scan const& scanref);
casacore::Int updateFieldTable_dynamic(casacore::MeasurementSet& msref, j2ms::Scan const& scanref);
// Updates the Processor subtable with a new row for a new passid
casacore::Int updateProcessorTable( casacore::MeasurementSet& msref,

Loading…
Cancel
Save