---

SimpleKLDocModel.hpp

Go to the documentation of this file.

/*==========================================================================
 *
 *  Original source copyright (c) 2001, Carnegie Mellon University.
 *  See copyright.cmu for details.
 *  Modifications copyright (c) 2002, University of Massachusetts.
 *  See copyright.umass for details.
 *
 *==========================================================================
*/

#ifndef _SIMPLEKLDOCMODEL_HPP
#define _SIMPLEKLDOCMODEL_HPP

#include "DocumentRep.hpp"
#include "Index.hpp"
#include "UnigramLM.hpp"


namespace SimpleKLParameter {
  enum SmoothMethod  {JELINEKMERCER=0, DIRICHLETPRIOR=1, ABSOLUTEDISCOUNT=2, 
                      TWOSTAGE=3};
 
  enum SmoothStrategy  {INTERPOLATE=0, BACKOFF=1};

  enum QueryUpdateMethod {MIXTURE = 0, DIVMIN=1, MARKOVCHAIN=2, RM1=3, RM2=4};

  enum adjustedScoreMethods {QUERYLIKELIHOOD = 1, CROSSENTROPY = 2, 
                               NEGATIVEKLD = 3};

  struct DocSmoothParam {
    enum SmoothMethod smthMethod;
    enum SmoothStrategy smthStrategy;
    double ADDelta;
    double JMLambda;
    double DirPrior;
  };

  static enum SmoothMethod defaultSmoothMethod = DIRICHLETPRIOR;
  static enum SmoothStrategy defaultSmoothStrategy = INTERPOLATE;
  static double defaultADDelta = 0.7;
  static double defaultJMLambda = 0.5;
  static double defaultDirPrior = 1000;

  struct QueryModelParam {
    enum adjustedScoreMethods adjScoreMethod;
    double qryNoise;

    enum QueryUpdateMethod fbMethod;
    double fbCoeff;
    int fbTermCount;
    double fbPrTh;
    double fbPrSumTh;
    double fbMixtureNoise;
    int emIterations;
  };

  static enum QueryUpdateMethod defaultFBMethod = MIXTURE;
  static double defaultFBCoeff = 0.5;
  static int defaultFBTermCount =50;
  static double defaultFBPrTh = 0.001;
  static double defaultFBPrSumTh = 1;
  static double defaultFBMixNoise = 0.5;
  static int defaultEMIterations = 50;
  static double defaultQryNoise = 0; //maximum likelihood estimator
};



class SimpleKLDocModel : public DocumentRep {
public:
  SimpleKLDocModel(DOCID_T docID, const UnigramLM &collectLM, int dl = 1, 
                   const double *prMass = NULL,
                   SimpleKLParameter::SmoothStrategy strat = SimpleKLParameter::INTERPOLATE) : 
    DocumentRep(docID, dl), 
    refLM(collectLM), docPrMass(prMass), strategy(strat) {
  };
  
  ~SimpleKLDocModel() {};

  virtual double termWeight(TERMID_T termID, const DocInfo *info) const {
    double sp = seenProb(info->termCount(), termID);
    double usp = unseenCoeff();
    double ref = refLM.prob(termID);
    double score = sp/(usp*ref);
    /*
    cerr << "TW:" << termID << " sp:" << sp << " usp:" << usp << " ref:" << ref << " s:" << score << endl;
    */
    //    return (seenProb(info->termCount(), termID)/(unseenCoeff()* refLM.prob(termID)));
    return score;
  }

  virtual double scoreConstant() const {
    return unseenCoeff();
  }

  virtual double unseenCoeff() const =0; // a(d)
  virtual double seenProb(double termFreq, TERMID_T termID) const =0;

protected:
  const UnigramLM &refLM;
  const double *docPrMass;
  SimpleKLParameter::SmoothStrategy strategy;
};




class JelinekMercerDocModel : public SimpleKLDocModel {
public:
  JelinekMercerDocModel(DOCID_T docID, 
                        int dl,
                        const UnigramLM &collectLM,
                        const double *docProbMass,
                        double collectLMWeight, 
                        SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM, dl, docProbMass, smthStrategy),
    lambda(collectLMWeight) {
  };

  virtual ~JelinekMercerDocModel() {};
  
  virtual double unseenCoeff() const {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return lambda;
    } else if (strategy==SimpleKLParameter::BACKOFF) {
      return lambda/(1-docPrMass[id]);
    } else {
      throw Exception("JelinekMercerDocModel", "Unknown smoothing strategy");
    }
  }
  virtual double seenProb(double termFreq, TERMID_T termID) const {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return ((1-lambda)*termFreq/(double)docLength +
              lambda*refLM.prob(termID));
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return ((1-lambda)*termFreq/(double)docLength);
    } else {
      throw Exception("JelinekMercerDocModel", "Unknown smoothing strategy");
    }
  }
private:
  double lambda;
};


class DirichletPriorDocModel : public SimpleKLDocModel {
public:
  DirichletPriorDocModel(DOCID_T docID,
                         int dl,
                         const UnigramLM &collectLM,
                         const double *docProbMass,
                         double priorWordCount,
                         SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM, dl, docProbMass, smthStrategy),
    mu(priorWordCount) {
  };

  virtual ~DirichletPriorDocModel() {};

  virtual double unseenCoeff() const {

    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return mu/(mu+docLength);
    } else if (strategy==SimpleKLParameter::BACKOFF) {
      return (mu/((mu+docLength)*(1-docPrMass[id])));
    } else {
      throw Exception("DirichletPriorDocModel", "Unknown smoothing strategy");
    }
  }

  virtual double seenProb(double termFreq, TERMID_T termID) const {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return (termFreq+mu*refLM.prob(termID))/
        (double)(docLength+mu);
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return (termFreq/(double)(docLength+mu));
    } else {      
      throw Exception("DirichletPriorDocModel", "Unknown smoothing strategy");
    }
  }
private:
  double mu;
};


class AbsoluteDiscountDocModel : public SimpleKLDocModel {
public:
  AbsoluteDiscountDocModel(DOCID_T docID,
                           int dl,
                           const UnigramLM &collectLM,
                           const double *docProbMass,
                           COUNT_T *uniqueTermCount,
                           double discount,
                           SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM, dl, docProbMass, smthStrategy),
    uniqDocLen(uniqueTermCount),
    delta(discount) {
  };

  virtual ~AbsoluteDiscountDocModel() {};
  
  virtual double unseenCoeff() const {

    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return (delta*uniqDocLen[id]/(double)docLength);
    } else if (strategy==SimpleKLParameter::BACKOFF) {
      return (delta*uniqDocLen[id]/(docLength*(1-docPrMass[id])));
    } else {
      throw Exception("AbsoluteDiscountDocModel", "Unknown smoothing strategy");
    }
  }
  virtual double seenProb(double termFreq, TERMID_T termID) const {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return ((termFreq-delta)/(double)docLength+
              delta*uniqDocLen[id]*refLM.prob(termID)/(double)docLength);
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return ((termFreq-delta)/(double)docLength);
    } else {
            throw Exception("AbsoluteDiscountDocModel", "Unknown smoothing strategy");
    }
  }
private:
  double *collectPr;
  COUNT_T *uniqDocLen;
  double delta;
};


// alpha = (mu+lambda*dLength)/(dLength+mu)
// pseen(w) = [(1-lambda)*c(w;d)+ (mu+lambda*dLength)*Pc(w)]/(dLength + mu)
class TwoStageDocModel : public SimpleKLDocModel {
public:
  TwoStageDocModel(DOCID_T docID,
                   int dl,
                   const UnigramLM &collectLM,
                   const double *docProbMass,
                   double firstStageMu, 
                   double secondStageLambda, 
                SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM, dl, docProbMass, smthStrategy),
    mu(firstStageMu),
    lambda(secondStageLambda) {
  };

  virtual ~TwoStageDocModel() {};

  virtual double unseenCoeff() const {

    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return (mu+lambda*docLength)/(mu+docLength);
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return ((mu+lambda*docLength)/((mu+docLength)*(1-docPrMass[id])));
    } else {
            throw Exception("TwoStageDocModel", "Unknown smoothing strategy");
    }
  }

  virtual double seenProb(double termFreq, TERMID_T termID) const {
    if (strategy == SimpleKLParameter::INTERPOLATE) {      
      return ((1-lambda)*(termFreq+mu*refLM.prob(termID))/
              (double)(docLength+mu) + lambda*refLM.prob(termID));
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return (termFreq*(1-lambda)/(double)(docLength+mu));
    } else {
            throw Exception("TwoStageDocModel", "Unknown smoothing strategy");
    }
  }
private:
  double mu;
  double lambda;
};

#endif /* _SIMPLEKLDOCMODEL_HPP */

---