---

BasicIndex.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.cs.cmu.edu/~lemur/license.html
 *
 *==========================================================================
*/


#ifndef _BASICINDEX_HPP
#define _BASICINDEX_HPP


//--------------------------------------------------------
// originally written by J. Lafferty, 11/00
//-----------------------------------------------------------------


#include <cstdio>
#include <ctime>
#include "common_headers.hpp"
#include "Compress.hpp"
#include "DocStream.hpp"
#include "FastList.hpp"
#include "IndexCount.hpp"
#include "MemList.hpp"
#include "String.hpp"
#include "Terms.hpp"
#include "Index.hpp"
#include "DocStream.hpp"

class BasicIndex : public Index {
public:
  BasicIndex();

  BasicIndex (Compress * pc);

  virtual ~BasicIndex();
  
  virtual bool open(const string &indexName);



  virtual int term (const string &word) const { return terms[word];}

  virtual const string term (int termID) const { return terms[termID];} 

  virtual int document (const string &docIDStr) const {return docids[docIDStr];}

  virtual const string document (int docID) const {return docids[docID];}

  virtual const string termLexiconID() const { return wordVocabulary;} 



  virtual int docCount () const  { return docids.size()-1;}

  virtual int termCountUnique () const { return terms.size()-1;}

  virtual int termCount (int termID) const { return countOfTerm[termID] ;}

  //  virtual int termCount () const {return (int)avgDocumentLength*(docids.size()-1) ;}
  virtual int termCount () const {return numWords ;}
  // XXX Better to have an exact count!

  virtual float docLengthAvg() const { return avgDocumentLength;}

  virtual int docCount(int termID) const;

  virtual int docLength (int docID) const { return countOfDoc[docID];}  ;




  virtual DocInfoList *docInfoList(int termID) const ;

  virtual TermInfoList *termInfoList(int docID) const ;



  // Create basic inverted indices 
  void build(DocStream *collectionStream,
             const string &file,
             const string &outputPrefix, 
             int totalDocs=0x1000000, int maxMemory=0x4000000,
             int minimumCount=1, int maxVocSize=2000000);
private:
  void buildVocabulary(int maxVocSize, int minimumCount);
  void writeWordIndex(int indexNum, FastList<IndexCount> * dlw);
  int indexCollection();
  int headDocIndex();
  int headWordIndex();
  void createKeys();
  void mergeIndexFiles();
  void createKey(const string &inName, const string &outName, 
                 Terms & voc, int * byteOffset);
  int mergePair(const string &fn1, const string &fn2, const string &fn3);
  void writeIndexFile();

  ifstream textStream;

  String      prefix;
  String      textFile;
  String      wordVocabulary;
  String      documentVocabulary;
  String      wordIndexFile, documentIndexFile;
  String      wordKeyFile, documentKeyFile;
  Terms       terms;
  Terms       docids;
  int         numDocuments;
  int         numWords;
  int         numBytes;
  int         maxDocumentLength;
  float       avgDocumentLength;
  int         totalDocuments;

  // The following are "utility" variables
private:
  int         memorySegment;
  int         maxSegmentsPerIndex;
  time_t      timeToIndex;
  int         maximumMemory;
  // int *       byteOffsetOfDoc;
  MemList *   pMemList;  
  Compress *  pCompressor;
  bool deleteCompressor;
  DocStream * pDocStream;

private:
  // fields for managing indices
  mutable ifstream  wordIndexStream, documentIndexStream;
  int * woffset, * doffset; //  *toffset;
  int * tmpdarr, * tmpwarr;
  int * countOfTerm;
  int * countOfDoc;

};

#endif

---