Main Page | Namespace List | Class Hierarchy | Class List | File List | Namespace Members | Class Members | File Members | Related Pages

indri::parse::Combiner Class Reference

#include <Combiner.hpp>

List of all members.

Public Member Functions

 Combiner (int bins=10)
void combineRedirectDestinationBuckets (const std::string &tmpPath)
void combineBuckets (const std::string &outputPath, const std::string &tmpPath)
void hashRedirectTargets (const std::string &bucketPath, const std::string &redirectsPath)
void hashToBuckets (const std::string &bucketPath, const std::string &inputPath)
void sortCorpusFiles (const std::string &outputPath, const std::string &preSortPath, const std::string &inputPath)

Private Types

typedef indri::utility::HashTable<
char *, url_entry *, strhash,
strcompst > 		UrlEntryTable
typedef indri::utility::HashTable<
char *, std::vector< url_entry * >,
strhash, strcompst > 		UrlEntryVectorTable

Private Member Functions

url_entry * _newUrlEntry (const char *url, const char *corpusPath, const char *docNo)
void _deleteUrlEntry (void *buffer)
void _readLinks (UrlEntryTable &urlTable, std::ifstream &linkIn)
void _readRedirects (UrlEntryTable &urlTable, const std::string &redirectPath, int number)
void _writeCorpusTable (UrlEntryVectorTable &corpusTable, const std::string &outputPath)
void _hashToCorpusTable (UrlEntryVectorTable &corpusTable, UrlEntryTable &urlTable)
void _openWriteBuckets (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets, const std::string &path, int bins)
void _flushWriteBuffer (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets, bool force, int i)
void _flushWriteBuffers (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets, bool force)
void _closeWriteBuckets (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets)
void _openReadBuckets (std::vector< std::ifstream * > &buckets, const std::string &path, int bins)
void _readDocBucket (UrlEntryTable &urlTable, std::ifstream &docIn)
int hashString (const char *str)
void hashToBuckets (std::ifstream &in, const std::string &path)
void createBuckets (const std::string &tmpPath)
void closeBuckets ()
void combineBucket (const std::string &outputPath, const std::string &tmpPath, int bucket)
void hashToBuckets (const std::string &inputPath)
void combineRedirectDestinationBucket (const std::string &tmpPath, int i, std::vector< std::stringstream * > &outBuffers, std::vector< std::ofstream * > &outputFiles)

Private Attributes

std::vector< std::ofstream * > _docBucketFiles
std::vector< std::ofstream * > _linkBucketFiles
std::vector< std::stringstream * > _docBuckets
std::vector< std::stringstream * > _linkBuckets
int _bins

Member Typedef Documentation

typedef indri::utility::HashTable<char*, url_entry*, strhash, strcompst> indri::parse::Combiner::UrlEntryTable [private]
 

typedef indri::utility::HashTable<char*, std::vector<url_entry*>, strhash, strcompst> indri::parse::Combiner::UrlEntryVectorTable [private]
 

Constructor & Destructor Documentation

indri::parse::Combiner::Combiner (  int  bins = 10  )  [inline]
 

Member Function Documentation

void indri::parse::Combiner::_closeWriteBuckets (  std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets
)  [private]
 

void indri::parse::Combiner::_deleteUrlEntry (  void *  buffer  )  [private]
 

void indri::parse::Combiner::_flushWriteBuffer (  std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets,
bool  force,
int  i
)  [private]
 

void indri::parse::Combiner::_flushWriteBuffers (  std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets,
bool  force
)  [private]
 

void indri::parse::Combiner::_hashToCorpusTable (  UrlEntryVectorTable &  corpusTable,
UrlEntryTable &  urlTable
)  [private]
 

indri::parse::Combiner::url_entry * indri::parse::Combiner::_newUrlEntry (  const char *  url,
const char *  corpusPath,
const char *  docNo
)  [private]
 

void indri::parse::Combiner::_openReadBuckets (  std::vector< std::ifstream * > &  buckets,
const std::string &  path,
int  bins
)  [private]
 

void indri::parse::Combiner::_openWriteBuckets (  std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets,
const std::string &  path,
int  bins
)  [private]
 

void indri::parse::Combiner::_readDocBucket (  UrlEntryTable &  urlTable,
std::ifstream &  docIn
)  [private]
 

void indri::parse::Combiner::_readLinks (  UrlEntryTable &  urlTable,
std::ifstream &  linkIn
)  [private]
 

void indri::parse::Combiner::_readRedirects (  UrlEntryTable &  urlTable,
const std::string &  redirectPath,
int  number
)  [private]
 

void indri::parse::Combiner::_writeCorpusTable (  UrlEntryVectorTable &  corpusTable,
const std::string &  outputPath
)  [private]
 

void indri::parse::Combiner::closeBuckets (   )  [private]
 

void indri::parse::Combiner::combineBucket (  const std::string &  outputPath,
const std::string &  tmpPath,
int  bucket
)  [private]
 

void indri::parse::Combiner::combineBuckets (  const std::string &  outputPath,
const std::string &  tmpPath
) 
 

void indri::parse::Combiner::combineRedirectDestinationBucket (  const std::string &  tmpPath,
int  i,
std::vector< std::stringstream * > &  outBuffers,
std::vector< std::ofstream * > &  outputFiles
)  [private]
 

void indri::parse::Combiner::combineRedirectDestinationBuckets (  const std::string &  tmpPath  ) 
 

void indri::parse::Combiner::createBuckets (  const std::string &  tmpPath  )  [private]
 

void indri::parse::Combiner::hashRedirectTargets (  const std::string &  bucketPath,
const std::string &  redirectsPath
) 
 

int indri::parse::Combiner::hashString (  const char *  str  )  [private]
 

void indri::parse::Combiner::hashToBuckets (  const std::string &  bucketPath,
const std::string &  inputPath
) 
 

void indri::parse::Combiner::hashToBuckets (  const std::string &  inputPath  )  [private]
 

void indri::parse::Combiner::hashToBuckets (  std::ifstream &  in,
const std::string &  path
)  [private]
 

void indri::parse::Combiner::sortCorpusFiles (  const std::string &  outputPath,
const std::string &  preSortPath,
const std::string &  inputPath
) 
 

Member Data Documentation

int indri::parse::Combiner::_bins [private]
 

std::vector< std::ofstream* > indri::parse::Combiner::_docBucketFiles [private]
 

std::vector< std::stringstream* > indri::parse::Combiner::_docBuckets [private]
 

std::vector< std::ofstream* > indri::parse::Combiner::_linkBucketFiles [private]
 

std::vector< std::stringstream* > indri::parse::Combiner::_linkBuckets [private]
 

The documentation for this class was generated from the following files:
Generated on Tue Jun 15 11:03:03 2010 for Lemur by doxygen 1.3.4