---

BasicIndex.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.cs.cmu.edu/~lemur/license.html
 *
 *==========================================================================
*/


#ifndef _BASICINDEX_HPP
#define _BASICINDEX_HPP


//--------------------------------------------------------
// originally written by J. Lafferty, 11/00
//-----------------------------------------------------------------


#include <cstdio>
#include <ctime>
#include "common_headers.hpp"
#include "Compress.hpp"
#include "DocStream.hpp"
#include "FastList.hpp"
#include "IndexCount.hpp"
#include "MemList.hpp"
#include "String.hpp"
#include "Terms.hpp"
#include "Index.hpp"
#include "DocStream.hpp"

class BasicIndex : public Index {
public:
  BasicIndex();

  BasicIndex (Compress * pc);

  virtual ~BasicIndex();
  
  virtual bool open(const char * indexName);



  virtual int term (const char * word) { return terms[word];}

  virtual const char * term (int termID) { return terms[termID];} 

  virtual int document (const char * docIDStr) {return docids[docIDStr];}

  virtual const char * document (int docID) {return docids[docID];}

  virtual const char *termLexiconID() { return wordVocabulary;} 



  virtual int docCount ()  { return docids.size()-1;}

  virtual int termCountUnique () { return terms.size()-1;}

  virtual int termCount (int termID) const { return countOfTerm[termID] ;}

  //  virtual int termCount () const {return (int)avgDocumentLength*(docids.size()-1) ;}
  virtual int termCount () const {return numWords ;}
  // XXX Better to have an exact count!

  virtual float docLengthAvg() { return avgDocumentLength;}

  virtual int docCount(int termID);

  virtual int docLength (int docID) const { return countOfDoc[docID];}  ;




  virtual DocInfoList *docInfoList(int termID) ;

  virtual TermInfoList *termInfoList(int docID) ;



  // Create basic inverted indices 
  void build(DocStream *collectionStream,
             const char *file,
             const char * outputPrefix, 
             int totalDocs=0x1000000, int maxMemory=0x4000000,
             int minimumCount=1, int maxVocSize=2000000);
private:
  void buildVocabulary(int maxVocSize, int minimumCount);
  void writeWordIndex(int indexNum, FastList<IndexCount> * dlw);
  int indexCollection();
  int headDocIndex();
  int headWordIndex();
  void createKeys();
  void mergeIndexFiles();
  void createKey(const char * inName, const char * outName, Terms & voc, int * byteOffset);
  int mergePair(const char * fn1, const char * fn2, const char * fn3);
  void writeIndexFile();

  ifstream textStream;

  String      prefix;
  String      textFile;
  String      wordVocabulary;
  String      documentVocabulary;
  String      wordIndexFile, documentIndexFile;
  String      wordKeyFile, documentKeyFile;
  Terms       terms;
  Terms       docids;
  int         numDocuments;
  int         numWords;
  int         numBytes;
  int         maxDocumentLength;
  float       avgDocumentLength;
  int         totalDocuments;

  // The following are "utility" variables
private:
  int         memorySegment;
  int         maxSegmentsPerIndex;
  time_t      timeToIndex;
  int         maximumMemory;
  // int *       byteOffsetOfDoc;
  MemList *   pMemList;  
  Compress *  pCompressor;
  bool deleteCompressor;
  DocStream * pDocStream;

private:
  // fields for managing indices
  ifstream  wordIndexStream, documentIndexStream;
  int * woffset, * doffset; //  *toffset;
  int * tmpdarr, * tmpwarr;
  int * countOfTerm;
  int * countOfDoc;

};

#endif

---