---

SimpleKLRetMethod.hpp

Go to the documentation of this file.

/*==========================================================================
 *
 *  Original source copyright (c) 2001, Carnegie Mellon University.
 *  See copyright.cmu for details.
 *  Modifications copyright (c) 2002, University of Massachusetts.
 *  See copyright.umass for details.
 *
 *==========================================================================
*/


#ifndef _SIMPLEKLRETMETHOD_HPP
#define _SIMPLEKLRETMETHOD_HPP

#include <cmath>
#include "UnigramLM.hpp"
#include "ScoreFunction.hpp"
#include "SimpleKLDocModel.hpp"
#include "TextQueryRep.hpp"
#include "TextQueryRetMethod.hpp"
#include "Counter.hpp"
#include "DocUnigramCounter.hpp"


class SimpleKLQueryModel : public ArrayQueryRep {
public:
  SimpleKLQueryModel(TextQuery &qry, Index &dbIndex) : ArrayQueryRep(dbIndex.termCountUnique()+1, qry, dbIndex), qm(NULL), ind(dbIndex), colKLComputed(false) {
  }

  SimpleKLQueryModel(Index &dbIndex) : ArrayQueryRep(dbIndex.termCountUnique()+1), qm(NULL), ind(dbIndex), colKLComputed(false) {
    startIteration();
    while (hasMore()) {
      QueryTerm *qt = nextTerm();
      setCount(qt->id(), 0);
      delete qt;
    }

  }


  virtual ~SimpleKLQueryModel(){ if (qm) delete qm;}



  virtual void interpolateWith(UnigramLM &qModel, double origModCoeff, int howManyWord, double prSumThresh=1, double prThresh=0);
  virtual double scoreConstant() {
    return totalCount();
  }
  
  virtual void load(istream &is);

  virtual void save(ostream &os);

  virtual void clarity(ostream &os);
  virtual double clarity();


  double colDivergence() {
    if (colKLComputed) {
      return colKL;
    } else {
      colKLComputed = true;
      double d=0;
      startIteration();
      while (hasMore()) {
        QueryTerm *qt=nextTerm();
        double pr = qt->weight()/(double)totalCount();
        double colPr = (ind.termCount(qt->id())+1)/(double)(ind.termCount()+ind.termCountUnique()); // Laplace smoothing, same as in SimpleKLRetMethod
        d += pr*log(pr/colPr);
        delete qt;
        
      }
      colKL=d;
      return d;
    }
  }



  double KLDivergence(UnigramLM &refMod) {
    double d=0;
    startIteration();
    while (hasMore()) {
      QueryTerm *qt=nextTerm();
      double pr = qt->weight()/(double)totalCount();
      d += pr*log(pr/refMod.prob(qt->id()));
      delete qt;
    }
    return d;
  }



protected:

  double colKL;
  bool colKLComputed;

  IndexedRealVector *qm;
  Index &ind;
};




class SimpleKLScoreFunc : public ScoreFunction {
public:

  virtual double matchedTermWeight(QueryTerm *qTerm, TextQueryRep *qRep, DocInfo *info, DocumentRep *dRep) { 
    return (qTerm->weight()*log(dRep->termWeight(qTerm->id(),info)));
  }

  virtual double adjustedScore(double origScore, TextQueryRep *qRep, DocumentRep *dRep) {
    SimpleKLQueryModel *qm = (SimpleKLQueryModel *)qRep;
    // dynamic_cast<SimpleKLQueryModel *>qRep;
    SimpleKLDocModel *dm = (SimpleKLDocModel *)dRep;
      // dynamic_cast<SimpleKLDocModel *>dRep;


    // this is the original query likelihood scoring formula
    //  return (origScore+log(dm->scoreConstant())*qm->scoreConstant());

    // This is the normalized query-likelihood, i.e., cross-entropy
    // assert(qm->scoreConstant()!=0);
    // return (origScore/qm->scoreConstant() + log(dm->scoreConstant()));

    // This is the exact (negative) KL-divergence value, i.e., -D(Mq||Md)
    assert(qm->scoreConstant()!=0);
    return (origScore/qm->scoreConstant() + log(dm->scoreConstant())
            - qm->colDivergence());


  }
};






class SimpleKLRetMethod : public TextQueryRetMethod {
public:

  SimpleKLRetMethod(Index &dbIndex, const char *supportFileName, ScoreAccumulator &accumulator);
  virtual ~SimpleKLRetMethod();
  
  virtual TextQueryRep *computeTextQueryRep(TextQuery &qry) {
    return (new SimpleKLQueryModel(qry, ind));
  }
  
  virtual DocumentRep *computeDocRep(int docID);
  

  virtual ScoreFunction *scoreFunc() {
    return (scFunc);
  }
  

  virtual void updateTextQuery(TextQueryRep &origRep, DocIDSet &relDocs);

  void setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam);
  void setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam);

protected:

  double *mcNorm; 
  
  double *docProbMass; 
  int *uniqueTermCount; 
  UnigramLM *collectLM; 
  DocUnigramCounter *collectLMCounter; 
  SimpleKLScoreFunc *scFunc; 



  void computeMixtureFBModel(SimpleKLQueryModel &origRep, DocIDSet & relDocs);
  void computeDivMinFBModel(SimpleKLQueryModel &origRep, DocIDSet &relDocs);
  void computeMarkovChainFBModel(SimpleKLQueryModel &origRep, DocIDSet &relDocs) ;
  void computeRM1FBModel(SimpleKLQueryModel &origRep, DocIDSet & relDocs);
  void computeRM2FBModel(SimpleKLQueryModel &origRep, DocIDSet & relDocs);

  SimpleKLParameter::DocSmoothParam docParam;
  SimpleKLParameter::QueryModelParam qryParam;

};


inline  void SimpleKLRetMethod::setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam)
{
  docParam = docSmthParam;
}

inline  void SimpleKLRetMethod::setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam)
{
  qryParam = queryModParam;
}

#endif /* _SIMPLEKLRETMETHOD_HPP */

---