UnigramLM.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */


#ifndef _UNIGRAMLM_HPP
#define _UNIGRAMLM_HPP

#include "Counter.hpp"
#include "Exception.hpp"
#include "IndexTypes.hpp"
#include <cstring>
namespace lemur 
{
  namespace langmod
  {
    

    class UnigramLM {
    public:
      virtual double prob(lemur::api::TERMID_T wordIndex) const = 0;
      virtual const string lexiconID() const= 0;

      virtual void startIteration() const = 0;
      virtual bool hasMore() const = 0;
      virtual void nextWordProb(lemur::api::TERMID_T &wordIndex, double &prob) const = 0;
    };



    class SmoothedMLEstimator : public UnigramLM {
    public:
      SmoothedMLEstimator(const lemur::utility::Counter &counter, const string &lexiconID) : ct(counter), lexID(lexiconID) {}
      virtual ~SmoothedMLEstimator() {}

      virtual double prob(lemur::api::TERMID_T wordIndex) const {
        return (probEstimate(wordIndex, ct.count(wordIndex),ct.sum()));
      }

      virtual void startIteration() const {
        ct.startIteration();
      }

      virtual bool hasMore() const {
        return ct.hasMore();
      }

      virtual void nextWordProb(lemur::api::TERMID_T &wordIndex, double &prob) const{
        double count;
        //dmf FIXME
        ct.nextCount((int&)wordIndex, count);
        prob = probEstimate(wordIndex, count, ct.sum());
      }
  
      virtual const string lexiconID() const { return lexID;}

      virtual double probEstimate(lemur::api::TERMID_T wordIndex, double wdCount, double sumCount) const=0;

    protected:
      const lemur::utility::Counter &ct;
      const string lexID;
    };
  

    class MLUnigramLM : public SmoothedMLEstimator { 
    public:
      MLUnigramLM(const lemur::utility::Counter & counter, const string &lexiconID) : SmoothedMLEstimator(counter, lexiconID) {};
      virtual ~MLUnigramLM() {}
  
      virtual double probEstimate(lemur::api::TERMID_T wordIndex, double count, double sum) const{
        return (count/sum);
      }
    };

    class LaplaceUnigramLM : public SmoothedMLEstimator { 
    public:
      LaplaceUnigramLM(const lemur::utility::Counter & counter, const string &lexiconID, double vocabSize) : SmoothedMLEstimator(counter, lexiconID), vocSz(vocabSize) {};
      virtual ~LaplaceUnigramLM() {}
  
      virtual double probEstimate(lemur::api::TERMID_T wordIndex, double count, double sum) const {
        return ((count+1)/(sum+vocSz));
      }
    private:
      double vocSz;
    };



    class DirichletUnigramLM : public SmoothedMLEstimator { 
    public:
      DirichletUnigramLM(const lemur::utility::Counter & counter, const string &lexiconID, 
                         const UnigramLM &refLM, double priorSampleSize) 
        : SmoothedMLEstimator(counter, lexiconID), ref(&refLM), 
          s(priorSampleSize) {}

      virtual ~DirichletUnigramLM() {}
  
      virtual double probEstimate(lemur::api::TERMID_T wordIndex, double count, double sum) const {
        return ((count+s*ref->prob(wordIndex))/(sum+s));
      }

    private:
      const UnigramLM *ref;
      double s;  
    };







    class InterpUnigramLM : public SmoothedMLEstimator { 
    public:
      InterpUnigramLM(const lemur::utility::Counter & counter, const string &lexiconID, 
                      const UnigramLM &refLM, double refCoeff) 
        : SmoothedMLEstimator(counter, lexiconID), ref(&refLM), 
          refC(refCoeff) {}

      virtual ~InterpUnigramLM() {}
  
      virtual double probEstimate(lemur::api::TERMID_T wordIndex, double count, double sum) const {
        return ((1-refC)*count/sum + refC*ref->prob(wordIndex));
      }

    private:
      const UnigramLM *ref;
      double refC;  
    };
 
  }
}
#endif /* _UNIGRAMLM_HPP */

---