---

SimpleKLDocModel.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.cs.cmu.edu/~lemur/license.html
 *
 *==========================================================================
*/


#ifndef _SIMPLEKLDOCMODEL_HPP
#define _SIMPLEKLDOCMODEL_HPP

#include "DocumentRep.hpp"
#include "Index.hpp"
#include "UnigramLM.hpp"


namespace SimpleKLParameter {
  enum SmoothMethod  {JELINEKMERCER=0, DIRICHLETPRIOR=1, ABSOLUTEDISCOUNT=2, TWOSTAGE=3};
 
  enum SmoothStrategy  {INTERPOLATE=0, BACKOFF=1};

  enum QueryUpdateMethod {MIXTURE = 0, DIVMIN=1, MARKOVCHAIN=2};

  struct DocSmoothParam {
    enum SmoothMethod smthMethod;
    enum SmoothStrategy smthStrategy;
    double ADDelta;
    double JMLambda;
    double DirPrior;
  };

  static enum SmoothMethod defaultSmoothMethod = DIRICHLETPRIOR;
  static enum SmoothStrategy defaultSmoothStrategy = INTERPOLATE;
  static double defaultADDelta = 0.7;
  static double defaultJMLambda = 0.5;
  static double defaultDirPrior = 1000;

  struct QueryModelParam {
    double qryNoise;

    enum QueryUpdateMethod fbMethod;
    double fbCoeff;
    int fbTermCount;
    double fbPrTh;
    double fbPrSumTh;
    double fbMixtureNoise;
    int emIterations;
  };

  static enum QueryUpdateMethod defaultFBMethod = MIXTURE;
  static double defaultFBCoeff = 0.5;
  static int defaultFBTermCount =50;
  static double defaultFBPrTh = 0.001;
  static double defaultFBPrSumTh = 1;
  static double defaultFBMixNoise = 0.5;
  static int defaultEMIterations = 50;
  static double defaultQryNoise = 0; //maximum likelihood estimator
};



class SimpleKLDocModel : public DocumentRep {
public:
  SimpleKLDocModel(int docID, UnigramLM &collectLM) : DocumentRep(docID), refLM(collectLM) {}
  ~SimpleKLDocModel() {};

  virtual double termWeight(int termID, DocInfo *info) {
    return (seenProb(info->termCount(), termID)/(unseenCoeff()* refLM.prob(termID)));
  }

  virtual double scoreConstant() {
    return unseenCoeff();
  }

  virtual double unseenCoeff()=0; // a(d)
  virtual double seenProb(double termFreq, int termID)=0; // p(w|d), w seen
protected:
  UnigramLM &refLM;
};




class JelinekMercerDocModel : public SimpleKLDocModel {
public:
  JelinekMercerDocModel(int docID, 
                        Index *referenceIndex, 
                        UnigramLM &collectLM,
                        double *docProbMass,
                        double collectLMWeight, 
                        SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM),
    refIndex(referenceIndex),
    docPrMass(docProbMass),
    lambda(collectLMWeight), 
    strategy(smthStrategy) {
  };

  virtual ~JelinekMercerDocModel() {};
  
  virtual double unseenCoeff() {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return lambda;
    } else if (strategy==SimpleKLParameter::BACKOFF) {
      return lambda/(1-docPrMass[id]);
    } else {
      throw Exception("JelinekMercerDocModel", "Unknown smoothing strategy");
    }
  }
  virtual double seenProb(double termFreq, int termID) {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return ((1-lambda)*termFreq/(double)refIndex->docLength(id)+
              lambda*refLM.prob(termID));
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return ((1-lambda)*termFreq/(double)refIndex->docLength(id));
    } else {
      throw Exception("JelinekMercerDocModel", "Unknown smoothing strategy");
    }
  }
private:
  Index *refIndex;
  double *docPrMass;
  double lambda;
  SimpleKLParameter::SmoothStrategy strategy;
};


class DirichletPriorDocModel : public SimpleKLDocModel {
public:
  DirichletPriorDocModel(int docID,
                   Index *referenceIndex, 
                   UnigramLM &collectLM,
                   double *docProbMass,
                   double priorWordCount, 
                SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM),
    refIndex(referenceIndex),
    docPrMass(docProbMass),
    mu(priorWordCount),
            strategy(smthStrategy) {
  };

  virtual ~DirichletPriorDocModel() {};

  virtual double unseenCoeff() {

    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return mu/(mu+refIndex->docLength(id));
    } else if (strategy==SimpleKLParameter::BACKOFF) {
      return (mu/((mu+refIndex->docLength(id))*
                      (1-docPrMass[id])));
    } else {
      throw Exception("DirichletPriorDocModel", "Unknown smoothing strategy");
    }
  }

  virtual double seenProb(double termFreq, int termID) {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return (termFreq+mu*refLM.prob(termID))/
        (double)(refIndex->docLength(id)+mu);
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return (termFreq/
              (double)(refIndex->docLength(id)+mu));
    } else {      
      throw Exception("DirichletPriorDocModel", "Unknown smoothing strategy");
    }
  }
private:
  Index *refIndex;
  double *docPrMass;
  double mu;
  SimpleKLParameter::SmoothStrategy strategy;
};


class AbsoluteDiscountDocModel : public SimpleKLDocModel {
public:
  AbsoluteDiscountDocModel(int docID,
                           Index *referenceIndex, 
                           UnigramLM &collectLM,
                           double *docProbMass,
                           int *uniqueTermCount,
                           double discount,
                SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM),
    refIndex(referenceIndex),
    docPrMass(docProbMass),
    uniqDocLen(uniqueTermCount),
    delta(discount),
      strategy(smthStrategy) {
  };

  virtual ~AbsoluteDiscountDocModel() {};
  
  virtual double unseenCoeff() {

    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return (delta*uniqDocLen[id]/(double)refIndex->docLength(id));
    } else if (strategy==SimpleKLParameter::BACKOFF) {
      return (delta*uniqDocLen[id]/
              (refIndex->docLength(id)*(1-docPrMass[id])));
    } else {
      throw Exception("AbsoluteDiscountDocModel", "Unknown smoothing strategy");
    }
  }
  virtual double seenProb(double termFreq, int termID) {
    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return ((termFreq-delta)/(double)refIndex->docLength(id)+
              delta*uniqDocLen[id]*refLM.prob(termID)/
              (double)refIndex->docLength(id));
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return ((termFreq-delta)/(double)refIndex->docLength(id));
    } else {
            throw Exception("AbsoluteDiscountDocModel", "Unknown smoothing strategy");
    }
  }
private:
  Index *refIndex;
  double *collectPr;
  double *docPrMass;
  int *uniqDocLen;
  double delta;
  SimpleKLParameter::SmoothStrategy strategy;
};



// alpha = (mu+lambda*dLength)/(dLength+mu)
// pseen(w) = [(1-lambda)*c(w;d)+ (mu+lambda*dLength)*Pc(w)]/(dLength + mu)
class TwoStageDocModel : public SimpleKLDocModel {
public:
  TwoStageDocModel(int docID,
                   Index *referenceIndex, 
                   UnigramLM &collectLM,
                   double *docProbMass,
                   double firstStageMu, 
                   double secondStageLambda, 
                SimpleKLParameter::SmoothStrategy smthStrategy=SimpleKLParameter::INTERPOLATE): 
    SimpleKLDocModel(docID, collectLM),
    refIndex(referenceIndex),
    docPrMass(docProbMass),
    mu(firstStageMu),
    lambda(secondStageLambda),
      strategy(smthStrategy) {
  };

  virtual ~TwoStageDocModel() {};

  virtual double unseenCoeff() {

    if (strategy == SimpleKLParameter::INTERPOLATE) {
      return (mu+lambda*refIndex->docLength(id))/(mu+refIndex->docLength(id));
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return ((mu+lambda*refIndex->docLength(id))
              /((mu+refIndex->docLength(id))*
                (1-docPrMass[id])));
    } else {
            throw Exception("TwoStageDocModel", "Unknown smoothing strategy");
    }
  }

  virtual double seenProb(double termFreq, int termID) {
    if (strategy == SimpleKLParameter::INTERPOLATE) {      
      return ((1-lambda)*(termFreq+mu*refLM.prob(termID))/
              (double)(refIndex->docLength(id)+mu) 
              + lambda*refLM.prob(termID));
    } else if (strategy == SimpleKLParameter::BACKOFF) {
      return (termFreq*(1-lambda)/
              (double)(refIndex->docLength(id)+mu));
    } else {
            throw Exception("TwoStageDocModel", "Unknown smoothing strategy");
    }
  }
private:
  Index *refIndex;
  double *docPrMass;
  double mu;
  double lambda;
  SimpleKLParameter::SmoothStrategy strategy;
};

#endif /* _SIMPLEKLDOCMODEL_HPP */

---