Browse Source

HV: * AriM sais that on Debian 9.2 ("Stretch") readdir_r(3)

is deprecated -> warning -> error -> no compile
      Rewritten to use readdir(3).
    * found issue with merging multithread search results:
      duplicate/failing inserts were not caught by 
      std::copy(this_thread_results.begin(), this_thread_results.end(),
                insert_iterator(totals))
      Rewritten to manual for(...) loop copying like an
      animal. But at least we can check each duplicate insert



git-svn-id: svn+ssh://code.jive.eu/code/svn/vbs_fs@57 5b2df0cb-e17b-44c8-90c8-480d528b6e0d
master
JIVE Software Dev 4 years ago
parent
commit
0fdce1504f
  1. 289
      directory_helper_templates.h
  2. 6
      vbs_fs.cc

289
directory_helper_templates.h

@ -103,8 +103,8 @@
// if( ptr->second )
// cout << ptr->first << " is younger than one minute" << endl;
//
#ifndef DIRECTORY_HELPER_TEMPLATES_H
#define DIRECTORY_HELPER_TEMPLATES_H
#ifndef EVLBI5A_DIRECTORY_HELPER_TEMPLATES_H
#define EVLBI5A_DIRECTORY_HELPER_TEMPLATES_H
#include <auto_array.h>
@ -135,37 +135,103 @@
///////////////////////////////////////////////
typedef std::set<std::string> direntries_type;
// provide a wrapper around ::strerror_r() such that
// users don't have to allocate a buffer themselves each time
// Note: of course this shouldn't /have/ to be a template
// but making it one means that it can live safely
// in a header-only "library" (like this one hopefully is :D)
template <size_t Size>
std::string errno_to_string(int e) {
char errBuf[Size];
::strerror_r(e, errBuf, Size);
return std::string(errBuf);
}
// A no-filter predicate - gets all entries in a directory
// this isn't a template and therefore it's left in comment
struct NoFilter {
bool operator()(std::string const&) const {
return true;
}
};
// Returns true if there exists at least one entry in dir for which
// Predicate return true. Stops as soon as it finds a match.
template <typename Predicate>
bool dir_exists(std::string const& dir, Predicate const& pred) {
// readdir_r(3) is deprecated so we use readdir(3) in stead.
// Because we use only stack variables in here we're thread safe by
// definition; it is impossible for two threads to be using the same
// DIR* being fed to readdir(3)
DIR* dirp;
bool rv = false;
struct dirent* entryPtr;
// This is a systemcall so can use ASSERT*() which will
// capture the error message from errno
if( (dirp=::opendir(dir.c_str()))==0 ) {
std::cerr << "*** dir_exists/failed to open " << dir << " - " << errno_to_string<256>(errno) << std::endl;
throw errno;
}
// Check each entry. Because readdir(3) does not return the error code,
// we follow the notes from readdir(3):
// set errno = 0 before the call, then check if NULL
// and use errno to disambiguate between error or end-of-directory
while( !rv ) {
errno = 0;
if( (entryPtr=::readdir(dirp))==0 )
break;
// skip . and ..
const std::string enm( entryPtr->d_name );
if( enm=="." || enm==".." )
continue;
// 'concatenate' the predicate's result to existing result
rv = (rv || pred(dir +"/" + enm));
}
// Force succesfull loop ending
const int oeno = errno;
::closedir(dirp);
if( oeno!=0 ) {
std::cerr << "*** dir_exists[" << dir << "] - " << errno_to_string<256>(oeno) << std::endl;
throw oeno;
}
return rv;
}
// This version will call the predicate with "entry->d_name"
// (i.e. just the name of the entry in the directory) as
// parameter because we do not know the path leading to DIR*
template <typename Predicate>
direntries_type dir_filter(DIR* dirp, Predicate const& pred) {
// Memory for the "dirent" struct may or may not include
// space for the "d_name[]" field. POSIX sais that it's
// almost impossible to pre-allocate the correct amount of memory
// but this is currently one of the better approximations
int eno;
const size_t entryLen = offsetof(struct dirent, d_name)+::pathconf("/", _PC_NAME_MAX) + 1;
struct dirent* entryPtr;
direntries_type rv;
auto_array<unsigned char> dirEntry( new unsigned char[entryLen] );
// readdir_r(3) is deprecated now so use readdir(3) in stead.
// Because we are given DIR*, we let someone else [the caller]
// worry about thread safety - i.e. the caller is responsible for
// NOT calling this method on the same DIR* from different threads
struct dirent* entryPtr;
direntries_type rv;
// Make sure it's rewound
::rewinddir( dirp );
// Check each entry
// ::readdir_r(3) returns 0 on success, entryPtr==NULL if end-of-directory reached
while( (eno=::readdir_r(dirp, (struct dirent*)&dirEntry[0], &entryPtr))==0 ) {
if( entryPtr==0 )
// Check each entry. Because readdir(3) does not return the error code,
// we follow the notes from readdir(3):
// set errno = 0 before the call, then check if NULL
// and use errno to disambiguate between error or end-of-directory
while( true ) {
errno = 0;
if( (entryPtr=::readdir(dirp))==0 )
break;
// If predicate returns true, add current entry to result
if( pred(std::string(entryPtr->d_name)) )
if( rv.insert(entryPtr->d_name).second==false )
std::cerr << "dir_filter[DIR*]/duplicate insert - " << entryPtr->d_name << std::endl;
}
if( eno!=0 ) {
std::cerr << "*** dir_filter: " << ::strerror(eno) << std::endl;
throw eno;
if( errno!=0 ) {
std::cerr << "*** dir_filter: " << errno_to_string<256>(errno) << std::endl;
throw errno;
}
return rv;
}
@ -175,108 +241,48 @@ direntries_type dir_filter(DIR* dirp, Predicate const& pred) {
// So it is "pred( dir + "/" + entry->d_name )"
template <typename Predicate>
direntries_type dir_filter(std::string const& dir, Predicate const& pred) {
// Memory for the "dirent" struct may or may not include
// space for the "d_name[]" field. POSIX sais that it's
// almost impossible to pre-allocate the correct amount of memory
// but this is currently one of the better approximations
int eno;
DIR* dirp;
const size_t entryLen = offsetof(struct dirent, d_name)+::pathconf("/", _PC_NAME_MAX) + 1;
struct dirent* entryPtr;
direntries_type rv;
auto_array<unsigned char> dirEntry( new unsigned char[entryLen] );
// readdir_r(3) is deprecated so we use readdir(3) in stead.
// Because we use only stack variables in here we're thread safe by
// definition; it is impossible for two threads to be using the same
// DIR* being fed to readdir(3)
DIR* dirp;
struct dirent* entryPtr;
direntries_type rv;
// This is a systemcall so can use ASSERT*() which will
// capture the error message from errno
if( (dirp=::opendir(dir.c_str()))==0 ) {
std::cerr << "*** dir_filter/failed to open " << dir << " - " << ::strerror(errno) << std::endl;
std::cerr << "*** dir_filter/failed to open " << dir << " - " << errno_to_string<256>(errno) << std::endl;
throw errno;
}
// Check each entry
// ::readdir_r(3) returns 0 on success, entryPtr==NULL if end-of-directory reached
while( (eno=::readdir_r(dirp, (struct dirent*)&dirEntry[0], &entryPtr))==0 ) {
if( entryPtr==0 )
// Check each entry. Because readdir(3) does not return the error code,
// we follow the notes from readdir(3):
// set errno = 0 before the call, then check if NULL
// and use errno to disambiguate between error or end-of-directory
while( true ) {
errno = 0;
if( (entryPtr=::readdir(dirp))==0 )
break;
// If predicate returns true, add current entry to result
if( pred(dir+"/"+entryPtr->d_name) )
if( rv.insert(dir+"/"+entryPtr->d_name).second==false )
std::cerr << "dir_filter[" << dir << "]/duplicate insert - " << entryPtr->d_name << std::endl;
}
// Force succesfull loop ending
int oeno = eno;
// Force succesfull loop ending; save current value of errno
// such that the result of closedir(3) does not clobber
// the result of processing the entries
const int oeno = errno;
::closedir(dirp);
if( oeno!=0 ) {
std::cerr << "*** dir_filter[" << dir << "] - " << ::strerror(oeno) << std::endl;
std::cerr << "*** dir_filter[" << dir << "] - " << errno_to_string<256>(oeno) << std::endl;
throw oeno;
}
return rv;
}
// Returns true if there exists at least one entry in dir for which
// Predicate return true. Stops as soon as it finds a match.
template <typename Predicate>
bool dir_exists(std::string const& dir, Predicate const& pred) {
// Memory for the "dirent" struct may or may not include
// space for the "d_name[]" field. POSIX sais that it's
// almost impossible to pre-allocate the correct amount of memory
// but this is currently one of the better approximations
int eno;
DIR* dirp;
bool rv = false;
const size_t entryLen = offsetof(struct dirent, d_name)+::pathconf("/", _PC_NAME_MAX) + 1;
struct dirent* entryPtr;
auto_array<unsigned char> dirEntry( new unsigned char[entryLen] );
// This is a systemcall so can use ASSERT*() which will
// capture the error message from errno
if( (dirp=::opendir(dir.c_str()))==0 ) {
std::cerr << "*** dir_exists/failed to open " << dir << " - " << ::strerror(errno) << std::endl;
throw errno;
}
// Check each entry until no more entries or a match
// ::readdir_r(3) returns 0 on success, entryPtr==NULL if end-of-directory reached
while( rv==false && (eno=::readdir_r(dirp, (struct dirent*)&dirEntry[0], &entryPtr))==0 ) {
if( entryPtr ) {
if( ::strcmp(entryPtr->d_name, ".")==0 || ::strcmp(entryPtr->d_name, "..")==0 )
continue;
rv = (rv || pred(dir+"/"+entryPtr->d_name));
} else
break;
}
// Force succesfull loop ending
int oeno = eno;
::closedir(dirp);
if( oeno!=0 ) {
std::cerr << "*** dir_exists[" << dir << "] - " << ::strerror(oeno) << std::endl;
throw oeno;
}
return rv;
}
#if 0
// A no-filter predicate - gets all entries in a directory
// this isn't a template and therefore it's left in comment
struct NoFilter {
bool operator()(std::string const&) const {
return true;
}
};
direntries_type vbs_readdir(DIR* dirp) {
return vbs_readdir(dirp, NoFilter());
}
direntries_type vbs_readdir(std::string const& dir) {
return vbs_readdir(dir, NoFilter());
}
#endif
///////////////////////////////////////////////
//
// The dir_mapper
@ -294,90 +300,101 @@ struct dir_mapper {
__m_cb( cb )
{}
// readdir_r(3) is deprecated so we use readdir(3) in stead.
// Because we use only stack variables in here we're thread safe by
// definition; it is impossible for two threads to be using the same
// DIR* being fed to readdir(3)
value_type operator()(std::string const& dir) const {
return (*this)(dir, NoFilter());
}
template <typename Predicate>
value_type operator()(std::string const& dir, Predicate p) {
int eno;
DIR* dirp;
value_type rv;
const size_t entryLen = offsetof(struct dirent, d_name)+::pathconf("/", _PC_NAME_MAX) + 1;
struct dirent* entryPtr;
auto_array<unsigned char> dirEntry( new unsigned char[entryLen] );
value_type operator()(std::string const& dir, Predicate const& p) const {
DIR* dirp;
value_type rv;
struct dirent* entryPtr;
// This is a systemcall so can use ASSERT*() which will
// capture the error message from errno
if( (dirp=::opendir(dir.c_str()))==0 ) {
std::cerr << "*** dir_mapper[" << dir << "] failed to opendir - " << ::strerror(errno) << std::endl;
std::cerr << "*** dir_mapper[" << dir << "] failed to opendir - " << errno_to_string<256>(errno) << std::endl;
throw errno;
}
// Check each entry
// ::readdir_r(3) returns 0 on success, entryPtr==NULL if end-of-directory reached
while( (eno=::readdir_r(dirp, (struct dirent*)&dirEntry[0], &entryPtr))==0 ) {
if( entryPtr==0 )
// Check each entry. Because readdir(3) does not return the error code,
// we follow the notes from readdir(3):
// set errno = 0 before the call, then check if NULL
// and use errno to disambiguate between error or end-of-directory
while( true ) {
errno = 0;
if( (entryPtr=::readdir(dirp))==0 )
break;
// If predicate returns true, add current entry to result
// skip . and .. and entries not fulfilling the predicate
const std::string d_name( entryPtr->d_name );
if( d_name=="." || d_name==".." || !p(d_name) )
continue;
// Now form complete name
const std::string fnm = dir + "/" + d_name;
// Request callback to yield a value for the current entry
if( rv.insert ( make_pair(d_name, __m_cb(fnm))).second==false )
std::cerr << "dir_mapper[" << dir << "]/duplicate insert - " << fnm << std::endl;
}
// Force succesfull loop ending
int oeno = eno;
// Force succesfull loop ending; save current value of errno
// such that the result of closedir(3) does not clobber
// the result of processing the entries
const int oeno = errno;
::closedir(dirp);
if( oeno!=0 ) {
std::cerr << "*** dir_mapper[" << dir << "] " << ::strerror(errno) << std::endl;
std::cerr << "*** dir_mapper[" << dir << "] " << errno_to_string<256>(oeno) << std::endl;
throw oeno;
}
return rv;
}
// readdir_r(3) is deprecated now so use readdir(3) in stead.
// Because we are given DIR*, we let someone else [the caller]
// worry about thread safety - i.e. the caller is responsible for
// NOT calling this method on the same DIR* from different threads
value_type operator()(DIR* dirp) const {
return (*this)(dirp, NoFilter());
}
template <typename Predicate>
value_type operator()(DIR* dirp, Predicate p) const {
int eno;
value_type rv;
const size_t entryLen = offsetof(struct dirent, d_name)+::pathconf("/", _PC_NAME_MAX) + 1;
struct dirent* entryPtr;
auto_array<unsigned char> dirEntry( new unsigned char[entryLen] );
value_type rv;
struct dirent* entryPtr;
::rewinddir(dirp);
// Check each entry
// ::readdir_r(3) returns 0 on success, entryPtr==NULL if end-of-directory reached
while( (eno=::readdir_r(dirp, (struct dirent*)&dirEntry[0], &entryPtr))==0 ) {
if( entryPtr==0 )
// Check each entry. Because readdir(3) does not return the error code,
// we follow the notes from readdir(3):
// set errno = 0 before the call, then check if NULL
// and use errno to disambiguate between error or end-of-directory
while( true ) {
errno = 0;
if( (entryPtr=::readdir(dirp))==0 )
break;
// If predicate returns true, add current entry to result
const std::string fnm = entryPtr->d_name;
const std::string fnm( entryPtr->d_name );
// Because we're working from DIR* we have no *idea* what the
// actual path is so all we can give the predicate and callback
// is the entry name.
// If the predicate returns false we're skipping this one
if( fnm=="." || fnm==".." || !p(fnm) )
continue;
if( rv.insert( make_pair(fnm, __m_cb(fnm))).second==false )
// Request callback to yield a value for the current entry
if( rv.insert ( make_pair(fnm, __m_cb(fnm))).second==false )
std::cerr << "dir_mapper[DIR*]/duplicate insert - " << fnm << std::endl;
}
if( eno!=0 ) {
std::cerr << "*** dir_mapper[DIR*] " << ::strerror(errno) << std::endl;
throw eno;
if( errno!=0 ) {
std::cerr << "*** dir_mapper[DIR*] " << errno_to_string<256>(errno) << std::endl;
throw errno;
}
return rv;
}
Callback __m_cb;
private:
struct NoFilter {
bool operator()(std::string const&) const {
return true;
}
};
};
#endif // DIRECTORY_HELPER_TEMPLATES_H

6
vbs_fs.cc

@ -724,7 +724,11 @@ void* doScan(void* ptr) {
// Now lock the result and transfer our results into that
scoped_lock_type sml( sfargs->mtx );
std::copy(tmpFC.begin(), tmpFC.end(), std::insert_iterator<filechunks_type>(*sfargs->fcPtr, sfargs->fcPtr->begin()));
// Manually copy the entries because we must check for #FAIL
// (std::copy to an output/insert iterator doesn't do that)
for(filechunks_type::const_iterator curFC = tmpFC.begin(); curFC!=tmpFC.end(); curFC++)
if( sfargs->fcPtr->insert(*curFC).second==false )
throw std::runtime_error(std::string("Duplicate insert of chunk ")+curFC->pathToChunk);
update(*sfargs->stPtr, tmpST);
}
catch( int e ) {

Loading…
Cancel
Save