---

SimpleKLRetMethod.hpp

Go to the documentation of this file.

/*==========================================================================
 *
 *  Original source copyright (c) 2001, Carnegie Mellon University.
 *  See copyright.cmu for details.
 *  Modifications copyright (c) 2002, University of Massachusetts.
 *  See copyright.umass for details.
 *
 *==========================================================================
*/


#ifndef _SIMPLEKLRETMETHOD_HPP
#define _SIMPLEKLRETMETHOD_HPP

#include <cmath>
#include "UnigramLM.hpp"
#include "ScoreFunction.hpp"
#include "SimpleKLDocModel.hpp"
#include "TextQueryRep.hpp"
#include "TextQueryRetMethod.hpp"
#include "Counter.hpp"
#include "DocUnigramCounter.hpp"


class SimpleKLQueryModel : public ArrayQueryRep {
public:
  SimpleKLQueryModel(TextQuery &qry, Index &dbIndex) : 
    ArrayQueryRep(dbIndex.termCountUnique()+1, qry, dbIndex), qm(NULL), 
    ind(dbIndex), colKLComputed(false) {
    startIteration();
    colQLikelihood = 0;
    //Sum w in Q qtf * log(qtcf/termcount);
    int tc = ind.termCount();
    while (hasMore()) {
      QueryTerm *qt = nextTerm();
      int id = qt->id();
      double qtf = qt->weight();
      int qtcf = ind.termCount(id);
      double s = qtf * log((double)qtcf/(double)tc);
      colQLikelihood += s;
      delete qt;
    }

  }

  SimpleKLQueryModel(Index &dbIndex) : 
    ArrayQueryRep(dbIndex.termCountUnique()+1), qm(NULL), ind(dbIndex), 
    colKLComputed(false) {
    colQLikelihood = 0;
    startIteration();
    while (hasMore()) {
      QueryTerm *qt = nextTerm();
      setCount(qt->id(), 0);
      delete qt;
    }
  }


  virtual ~SimpleKLQueryModel(){ if (qm) delete qm;}



  virtual void interpolateWith(UnigramLM &qModel, double origModCoeff, 
                               int howManyWord, double prSumThresh=1, 
                               double prThresh=0);
  virtual double scoreConstant() {
    return totalCount();
  }
  
  virtual void load(istream &is);

  virtual void save(ostream &os);

  virtual void clarity(ostream &os);
  virtual double clarity();

#if 0

  double colDivergence() {
    if (colKLComputed) {
      return colKL;
    } else {
      colKLComputed = true;
      double d=0;
      startIteration();
      while (hasMore()) {
        QueryTerm *qt=nextTerm();
        double pr = qt->weight()/(double)totalCount();
        double colPr = (ind.termCount(qt->id())+1)/(double)(ind.termCount()+ind.termCountUnique()); // Laplace smoothing, same as in SimpleKLRetMethod
        d += pr*log(pr/colPr);
        delete qt;
        
      }
      colKL=d;
      return d;
    }
  }
#endif

  double colDivergence() {
    if (colKLComputed) {
      return colKL;
    } else {
      colKLComputed = true;
      double d=0;
      startIteration();
      while (hasMore()) {
        QueryTerm *qt=nextTerm();
        double pr = qt->weight()/(double)totalCount();
        //      double colPr = (ind.termCount(qt->id())+1)/(double)(ind.termCount()+ind.termCountUnique()); // Laplace smoothing, same as in SimpleKLRetMethod
        double colPr = ((double)ind.termCount(qt->id())/(double)(ind.termCount())); // ML smoothing, same as in SimpleKLRetMethod
        d += pr*log(pr/colPr);
        delete qt;
        
      }
      colKL=d;
      return d;
    }
  }


  double KLDivergence(UnigramLM &refMod) {
    double d=0;
    startIteration();
    while (hasMore()) {
      QueryTerm *qt=nextTerm();
      double pr = qt->weight()/(double)totalCount();
      d += pr*log(pr/refMod.prob(qt->id()));
      delete qt;
    }
    return d;
  }

  double colQueryLikelihood() {
    return colQLikelihood;
  }
  

protected:
  // For Query likelihood adjusted score
  double colQLikelihood;
  
  double colKL;
  bool colKLComputed;

  IndexedRealVector *qm;
  Index &ind;
};




class SimpleKLScoreFunc : public ScoreFunction {
public:
  enum SimpleKLParameter::adjustedScoreMethods adjScoreMethod;
  void setScoreMethod(enum SimpleKLParameter::adjustedScoreMethods adj) {
    adjScoreMethod = adj;
  }  
  virtual double matchedTermWeight(QueryTerm *qTerm, TextQueryRep *qRep, DocInfo *info, DocumentRep *dRep) { 
    double w = qTerm->weight();
    double d = dRep->termWeight(qTerm->id(),info);
    double l = log(d);
    double score = w*l;
    /*
    cerr << "M:" << qTerm->id() <<" d:" << info->docID() << " w:" << w 
         << " d:" << d << " l:" << l << " s:" << score << endl;
    */
    return score;
    //    return (qTerm->weight()*log(dRep->termWeight(qTerm->id(),info)));
  }
  virtual double adjustedScore(double origScore, TextQueryRep *qRep, 
                               DocumentRep *dRep) {
    SimpleKLQueryModel *qm = (SimpleKLQueryModel *)qRep;
    // dynamic_cast<SimpleKLQueryModel *>qRep;
    SimpleKLDocModel *dm = (SimpleKLDocModel *)dRep;
      // dynamic_cast<SimpleKLDocModel *>dRep;

    double qsc = qm->scoreConstant();
    double dsc = log(dm->scoreConstant());
    double cql = qm->colQueryLikelihood();
    // real query likelihood
    double s = dsc * qsc + origScore + cql;
    double qsNorm = origScore/qsc;
    double qmD = qm->colDivergence();
    switch (adjScoreMethod) {
    case SimpleKLParameter::QUERYLIKELIHOOD:
      // this is the original query likelihood scoring formula
      /*
      cerr << "A:"<< origScore << " dsc:" << dsc  << " qsc:" << qsc  
           << " cql:" << cql << " s:"  << s << endl;
      */
      return s;
      //      return (origScore+log(dm->scoreConstant())*qm->scoreConstant());
    case SimpleKLParameter::CROSSENTROPY:
      // This is the normalized query-likelihood, i.e., cross-entropy
      assert(qm->scoreConstant()!=0);
      // return (origScore/qm->scoreConstant() + log(dm->scoreConstant()));
      // add the term colQueryLikelihood/qm->scoreConstant
      s = qsNorm + dsc + cql/qsc;
      return (s);
    case SimpleKLParameter::NEGATIVEKLD:
      // This is the exact (negative) KL-divergence value, i.e., -D(Mq||Md)
      assert(qm->scoreConstant()!=0);
      s = qsNorm + dsc - qmD;
      /*
      cerr << origScore << ":" << qsNorm << ":" << dsc  << ":" << qmD  << ":" << s << endl;
      */
      return s;
      //      return (origScore/qm->scoreConstant() + log(dm->scoreConstant())
      //              - qm->colDivergence());
    }
  }
  
#if 0

  virtual double adjustedScore(double origScore, TextQueryRep *qRep, DocumentRep *dRep) {
    SimpleKLQueryModel *qm = (SimpleKLQueryModel *)qRep;
    // dynamic_cast<SimpleKLQueryModel *>qRep;
    SimpleKLDocModel *dm = (SimpleKLDocModel *)dRep;
      // dynamic_cast<SimpleKLDocModel *>dRep;


    // this is the original query likelihood scoring formula
    //  return (origScore+log(dm->scoreConstant())*qm->scoreConstant());

    // This is the normalized query-likelihood, i.e., cross-entropy
    // assert(qm->scoreConstant()!=0);
    // return (origScore/qm->scoreConstant() + log(dm->scoreConstant()));

    // This is the exact (negative) KL-divergence value, i.e., -D(Mq||Md)
    assert(qm->scoreConstant()!=0);
    return (origScore/qm->scoreConstant() + log(dm->scoreConstant())
            - qm->colDivergence());


  }
#endif
};






class SimpleKLRetMethod : public TextQueryRetMethod {
public:

  SimpleKLRetMethod(Index &dbIndex, const char *supportFileName, ScoreAccumulator &accumulator);
  virtual ~SimpleKLRetMethod();
  
  virtual TextQueryRep *computeTextQueryRep(TextQuery &qry) {
    return (new SimpleKLQueryModel(qry, ind));
  }
  
  virtual DocumentRep *computeDocRep(int docID);
  

  virtual ScoreFunction *scoreFunc() {
    return (scFunc);
  }
  

  virtual void updateTextQuery(TextQueryRep &origRep, DocIDSet &relDocs);

  void setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam);
  void setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam);

protected:

  double *mcNorm; 
  
  double *docProbMass; 
  int *uniqueTermCount; 
  UnigramLM *collectLM; 
  DocUnigramCounter *collectLMCounter; 
  SimpleKLScoreFunc *scFunc; 



  void computeMixtureFBModel(SimpleKLQueryModel &origRep, DocIDSet & relDocs);
  void computeDivMinFBModel(SimpleKLQueryModel &origRep, DocIDSet &relDocs);
  void computeMarkovChainFBModel(SimpleKLQueryModel &origRep, DocIDSet &relDocs) ;
  void computeRM1FBModel(SimpleKLQueryModel &origRep, DocIDSet & relDocs);
  void computeRM2FBModel(SimpleKLQueryModel &origRep, DocIDSet & relDocs);

  SimpleKLParameter::DocSmoothParam docParam;
  SimpleKLParameter::QueryModelParam qryParam;

};


inline  void SimpleKLRetMethod::setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam)
{
  docParam = docSmthParam;
}

inline  void SimpleKLRetMethod::setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam)
{
  qryParam = queryModParam;
  // add a parameter to the score function.
  // isn't available in the constructor.
  scFunc->setScoreMethod(qryParam.adjScoreMethod);
}

#endif /* _SIMPLEKLRETMETHOD_HPP */

---