Initial support for "any" size FITS-IDI files

issue-6
haavee 9 months ago
parent 4dc2e4896d
commit 852e00550d
  1. 35
      apps/tConvert/tConvert.cc
  2. 2
      jive/j2ms/JIVEMSFiller.cc
  3. 6
      jive/ms2uvfitsimpl/Converters/FileFormatConverter.cc
  4. 3
      jive/ms2uvfitsimpl/Converters/FileFormatConverter.h
  5. 72
      jive/ms2uvfitsimpl/Converters/ms2idifitsConverter.cc
  6. 3
      jive/ms2uvfitsimpl/Converters/ms2idifitsConverter.h
  7. 4
      jive/ms2uvfitsimpl/FITSUtils/FITSDataSegment.cc

@ -120,14 +120,19 @@ bool readLisFile( const char* lisfile, job_t& job ) {
static const char* default_conversion = "m2i";
void Usage( const String& prgname ) {
cout << "Usage: " << prgname << " [option] <source> <destination>" << endl;
cout << " where option can be:" << endl;
cout << " -m2i -> conversion from MS (source) to IDI FITSfile" << endl
<< " (VLBA format)" << endl;
cout << " -wm2i -> conversion from Westerbork MS (source)" << endl
<< " to IDI FITSfile (destination)" << endl;
cout << " -fuzz SECONDS -> Extend scans for up to SECONDS at each end" << endl;
cout << " -v <lisfile> -> Read input MS, output FITS file name from .lis file" << endl;
cout << "Usage: " << prgname << " [options] <source> <destination>" << endl;
cout << " where options can be:" << endl;
cout << " -m2i conversion from MS (source) to IDI FITSfile" << endl
<< " (VLBA format)" << endl << endl;
cout << " -wm2i conversion from Westerbork MS (source)" << endl
<< " to IDI FITSfile (destination)" << endl << endl;
cout << " -fuzz SECONDS extend scans for up to SECONDS at each end" << endl << endl;
cout << " -v <lisfile> -> read input MS, output FITS file name from .lis file" << endl;
cout << " -o OPTION=VALUE to pass options to the conversion routine." << endl
<< " Currently supported OPTIONs:" << endl
<< " chunk_size=<number>([MGT]i?B)" << endl
<< " without suffix the chunk size is in bytes;" << endl
<< " MGT are base 1000 (B) or 1024 (iB) ^2, ^3, ^4" << endl;
cout << endl
<< "Defaults are: '" << default_conversion << "'" << endl;
return;
@ -149,6 +154,7 @@ typedef enum _runmode_t {
int main(int, char const*const* argv ) {
String infile;
String outfile;
String options;
String method( default_conversion );
runmode_t runmode = run_unknown;
@ -203,6 +209,17 @@ int main(int, char const*const* argv ) {
if( opt=="m2i" || opt=="wm2i" )
method = opt;
else if( opt=="o" ) {
// add next option to conversion option
if( *(++argv) ) {
if( !options.empty() )
options += ";";
options += *argv;
} else {
cout << "Missing converter option after -o option" << endl;
return -1;
}
}
else if( opt=="fuzz" ) {
if( *(++argv) )
MSConversionInfo::setFuzz( ::atof(*argv) );
@ -290,7 +307,7 @@ int main(int, char const*const* argv ) {
ms2idifitsConverter converter( hdustodo );
// Attempt the transformation....
if( !converter.convert(outfile, infile) ) {
if( !converter.convert(outfile, infile, options) ) {
cout << "Rats. Tranformation not succeeded" << endl;
return -1;
}

@ -160,7 +160,7 @@ JIVEMSFiller::JIVEMSFiller( const String& opt ):
string optcopy;
string::iterator curchar = myOptions.begin();
while( curchar<myOptions.end() ) {
while( curchar!=myOptions.end() ) {
if( !::strchr(" \t\n\f", *curchar) ) {
optcopy.append( 1, *curchar );
}

@ -36,11 +36,15 @@ FileFormatConverter::FileFormatConverter()
//
// The base class convert-method: do not convert; copy!
//
Bool FileFormatConverter::convert( const String& outputfilename, const String& inputfilename ) const
Bool FileFormatConverter::convert( const String& outputfilename, const String& inputfilename, const String& options ) const
{
Bool retval( False );
RegularFile infile( inputfilename );
if( !options.empty() ) {
cerr << "FileFormatConverter does not support passing of options ('" << options << "')" << endl;
return False;
}
try
{
infile.copy( outputfilename, False );

@ -20,7 +20,8 @@ public:
// force implementors to implement this'un
virtual bool convert( const casacore::String& outputfilename,
const casacore::String& inputfilename ) const = 0;
const casacore::String& inputfilename,
const casacore::String& options = "" ) const = 0;
// The destructor.. release stuff we have allocated (if any)
virtual ~FileFormatConverter();

@ -37,6 +37,7 @@
#include <casacore/tables/TaQL/ExprNode.h>
#include <jive/Utilities/streamutil.h>
#include <jive/Utilities/stringutil.h>
#include <sstream>
@ -48,9 +49,9 @@ using namespace casacore;
// Note:
uLong nrVisInChunk(MSConversionInfo const& base,
double const max_size_per_chunk = (double)(((unsigned)1)<<31) - 200*(1024*1024) /*allow for ~200MB overhead per chunk?*/ ) {
double size_per_row;
double overhead_per_row;
double size_per_visibility;
double size_per_row;
double overhead_per_row;
double size_per_visibility;
// Ok. From the MSConversionInfo we can derive the
// approx size of a row in the MS...
// Then we can easily work out how many rows of
@ -111,7 +112,66 @@ ms2idifitsConverter::ms2idifitsConverter( const generatorlist_t& hdus ):
typedef std::list<hduGenerator*> genpointers_type;
// This function really converts the MS into a UVFITS file....
bool ms2idifitsConverter::convert( const String& fitsfilename,
const String& MSfilename ) const {
const String& MSfilename,
const String& options ) const {
// Process options given, if any
string optcopy;
string::const_iterator curchar = options.begin();
while( curchar!=options.end() ) {
if( !::strchr(" \t\n\f", *curchar) ) {
optcopy.append( 1, *curchar );
}
curchar++;
}
// Step one: split the options at semicolons ';' -> they're the delimiters
// between separate options.
uLong chunk_size = 0L;
Regex rx_size( "^chunk_size=([0-9]+)([MGT]i?B)?$" );
vector<string> opts;
vector<string>::iterator curopt;
opts = ::split( optcopy, ';' );
curopt = opts.begin();
while( curopt!=opts.end() ) {
String tmp( *curopt );
if( tmp.matches(rx_size) ) {
Int s, l;
string sz, unit;
// size is the first matchgroup
rx_size.match_info(s, l, 1);
sz = tmp(s, l);
chunk_size = ::strtoul(sz.c_str(), NULL, 10);
// and the actual options are the second matchgroup
rx_size.match_info(s, l, 2);
unit = tmp(s, l);
if( !unit.empty() ) {
// base 1000 or 1024
uLong const base( unit.find("iB")!=string::npos ? 1024L : 1000L );
// It's either M, G, or T, so at least base^2 (M)
chunk_size *= base;
chunk_size *= base;
if( unit[0]=='G' || unit[0]=='T' )
chunk_size *= base;
if( unit[0]=='T' )
chunk_size *= base;
}
} else if( curopt->size() ) {
std::ostringstream e;
e << "ms2idifits: unsupported option `" << *curopt << "'";
throw std::runtime_error(e.str());
}
// Do *NOT* forget to move on! (Guess (again) how I found this one out...)
curopt++;
}
// Do some preprocessing on the HDU generator list of the HDUs we're
// supposed to generate. Make sure there is an actual generator
// registered for each type.
@ -183,8 +243,8 @@ bool ms2idifitsConverter::convert( const String& fitsfilename,
cout << "ms2idi: no HDUs generated by generators: " << noHDUs << std::endl;
// Need to work out how many visibilities per FITS file. At this
// point we make FITS files <= 2GB
uLong const nVisPerChunk = nrVisInChunk( conversion_info );
// The compiled in default max chunk size for nrVisInChunk() is 2 GB
uLong const nVisPerChunk = (chunk_size == 0L ? nrVisInChunk( conversion_info ) : nrVisInChunk( conversion_info, chunk_size ));
Block<String> iternames( 4 );
std::ostringstream filenamestrm;

@ -56,7 +56,8 @@ public:
// Do convert the MS into a UVFITS-file
bool convert( const casacore::String& fitsfilename,
const casacore::String& MSfilename ) const;
const casacore::String& MSfilename,
const casacore::String& options="" ) const;
// Replace the current generators by those listed in the
// block. The conversion class will set up the engine in such a

@ -114,10 +114,12 @@ FITSDataSegment::FITSDataSegment( ByteIO* bytestream, uLong size, DataType basic
// blocks...
itsFitsSize = (itsOriginalSize > 0) ? ((itsOriginalSize + 2880 - 1) / 2880) * 2880 : itsOriginalSize;
#if 0
// Check if we're not crossing the 2**31-1 byte boundary
if( itsFitsSize>=((1LL<<31)-1) )
throw( AipsError("FITSDataSegment: Attempt to create segment with size >= 2**31-1 bytes. Unsupported by the OS") );
#endif
// If the stream is not seekable, we might just want to copy
// this to a tempfile on disk (which is seekable!)
// Note June 2018: This was basically always done, only based on

Loading…
Cancel
Save