PDict.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2004 University of Massachusetts.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */


/* 
 * dmf 02/2004
 */
#ifndef _LEMUR_PDICT_HPP
#define _LEMUR_PDICT_HPP

#include "common_headers.hpp"
#include "algorithm"
#include "Keyfile.hpp"
#include "File.hpp"
#include "TextHandlerManager.hpp"
namespace lemur 
{
  namespace dictionary 
  {
    
    class DictEntry {
    public:
      DictEntry();

      DictEntry(const string &targ, const string &typ, double pr) : target(targ), type(typ), 
                                                                    prob(pr) { }
  
      char *toBytes(int &numBytes) const;

      int toBytes(char *buffer) const;

      int numBytes() const;
      int fromBytes(char *buffer);
      bool operator==(const DictEntry & a) const { 
        return (target == a.target && type == a.type); 
      }
      string toString(string delim = ";") const ;
    
      string target;
      string type;
      double prob;
    };

    class DictEntryFilter {
    public:
      virtual bool accept(const DictEntry &entry) const = 0;
      virtual ~DictEntryFilter() {}
    } ;


    class AllDictEntryFilter : public DictEntryFilter {
    public:
      bool accept(const DictEntry &entry) const { return true; }
    };


    class ProbDictEntryFilter : public DictEntryFilter {
    public:
      ProbDictEntryFilter(double thresh = 0.0) : threshold(thresh) {
      }
      bool accept(const DictEntry &entry) const { return entry.prob > threshold; }
    private:
      double threshold;
    };


    class TypeDictEntryFilter : public DictEntryFilter {
    public:
      TypeDictEntryFilter(const string &filtType) : type(filtType) {
      }
      bool accept(const DictEntry &entry) const { return entry.type == type; }
    private:
      string type;
    };

    class StopwordDictEntryFilter : public DictEntryFilter {
    public:
      StopwordDictEntryFilter(const string &stopwords) {
        stopper = lemur::api::TextHandlerManager::createStopper(stopwords);
      }
      bool accept(const DictEntry &entry) const { 
        return !(stopper->stopWord(entry.target.c_str())); 
      }
    private:
      lemur::api::Stopper *stopper;
    };

    class DictEntryVector : public vector<DictEntry> {
    public:
      DictEntryVector() : vector<DictEntry>() {
      }
      DictEntryVector(char *buffer, DictEntryFilter *filter);
      void sortScores() {
        sort(this->begin(), this->end(), cmpFn);
      }
      bool addEntry(DictEntry &entry, double (*compose)(double, double) = NULL);

      bool removeEntry(DictEntry &entry);

      char *toBytes(int &numBytes) const;

      void toBytes(char *buffer) const;

      void fromBytes(char *buffer, DictEntryFilter *filter);

      int numEntries() const;

      void normalize();
  
    private:
      class DictEntryProbDescending { 
      public: 
        bool operator()(const DictEntry & a, const DictEntry & b) {
          return a.prob > b.prob;
        }
      };
      static DictEntryProbDescending cmpFn;
    };

    struct dictStats {
      int dictSize;
      int sourceSize;
      int targetSize;
    };

    class PDict {
    public:
      PDict();
  
      ~PDict();

      DictEntryVector *getTranslations(const string &term, 
                                       DictEntryFilter *filter=NULL) const ;
      int numTranslations(const string &term, 
                          DictEntryFilter *filter=NULL) const;
      int getNumPairs() const;

      int getSourceCount() const;

      int getTargetCount() const ;

      const string &getName() const {return name;}

      bool isUsingCounts() const {return usingCounts;}
  
      void setUsingCounts(bool val) {usingCounts = val;}
  
      void add(const string &source, DictEntry &value, 
               double (*compose)(double, double) = NULL);

      void remove(const string &source, DictEntry &value);

      void remove(const string &source);

      void write(const string &outputName, const string &delim);

      bool read(const string &dictName, const string &delim, bool counts = false);

      bool open(const string &dictName);

      bool create(const string &dictName);

      void close();
  
      void normalize();

      void startIteration() {dict.setFirst();}

      DictEntryVector *nextTranslations(string &term, 
                                        DictEntryFilter *filter=NULL) const;

    private:
      void writeTOC() const;  
      bool contains(const string &term, lemur::file::Keyfile &keyfile) const;
      void flush();
      dictStats stats;
      DictEntryVector* currentVec;
      bool usingCounts;
      string currentTerm;
      string name;
      mutable lemur::file::Keyfile dict;
      mutable lemur::file::Keyfile targetIDs;
      mutable lemur::file::File dictEntries;
    };
  }
}

#endif // _LEMUR_PDICT_HPP

---