---

KeyfileIncIndex.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.cs.cmu.edu/~lemur/license.html
 *
 *==========================================================================
*/


#ifndef _LEMUR_KEYFILE_INCINDEX_HPP
#define _LEMUR_KEYFILE_INCINDEX_HPP

/*
 * NAME DATE - COMMENTS
 * tnt 01/02 - created
 * dmf 07/03 - converted to incremental berkeley db btree indexer with
 * integrated document manager.
 * tds 09/03 - modified from BTIncIndex to use keyfile
 * dmf 12/03 - update to 2.1 API and remove parser/docmgr components.
*/
#include "common_headers.hpp"
#include "Index.hpp"
#include "InvFPDocList.hpp"
#include "InvFPTermList.hpp"
#include "InvFPTypes.hpp"
#include "Param.hpp"
#include "PushIndex.hpp"
#include "MemCache.hpp"
#include "Keyfile.hpp"
#include "KeyfileDocMgr.hpp"
#include "ReadBuffer.hpp"
#include "WriteBuffer.hpp"
#include "TermCache.hpp"
#include <cstring>
#include <queue>

// for counts array
#define UNIQUE_TERMS 0
#define TOTAL_TERMS  1
#define DOCS         2
#define DT_FILES     3
#define INV_FILES    4
#define MAX_DOCID_LENGTH 256
#define MAX_TERM_LENGTH 256

#define KEYFILE_MAX_SEGMENTS (16)

// we love multiple inheritance

class KeyfileIncIndex : public PushIndex, public Index {
public:
  class record {
  public:
    File::offset_type offset;
    int len;
    int totalLen;
    int num;     
  };
  struct SegmentOffset {
    unsigned int segment;
    unsigned int length;
    File::offset_type offset;
  };
  struct TermData {
    unsigned int totalCount;
    unsigned int documentCount;
    SegmentOffset segments[ KEYFILE_MAX_SEGMENTS ];
  };
  KeyfileIncIndex(const char* indexName = 0); 
  KeyfileIncIndex(char* prefix, int cachesize=128000000, 
                  DOCID_T startdocid=1);
  ~KeyfileIncIndex();

  void setName(char* prefix);

  bool beginDoc(DocumentProps* dp);

  bool addTerm(Term& t);

  void endDoc(DocumentProps* dp);

  virtual void endDoc(DocumentProps* dp, const char* mgr);

  void endCollection(CollectionProps* cp);

  void setDocManager(const char* mgrID);
    
protected:
  bool tryOpen();
  void writeTOC();
  void writeCache( bool lastRun = false );
  void lastWriteCache();

  void mergeCacheSegments();
  void writeCacheSegment();
  void writeDocMgrIDs();
  int docMgrID(const char* mgr);
  virtual void doendDoc(DocumentProps* dp, int mgrid);
  int listlengths;
  
public:


  bool open(const char* indexName);



  int term(const char* word);

  const char* term(int termID);

  int document(const char* docIDStr);

  const char* document(int docID); 

  DocumentManager *docManager(int docID);




  int docCount() { return counts[DOCS]; };

  int termCountUnique() { return counts[UNIQUE_TERMS]; };

  int termCount(int termID) const;

  int termCount() const { return counts[TOTAL_TERMS]; };

  float docLengthAvg();

  int docCount(int termID);

  int docLength(DOCID_T docID) const; // should use DOCID_T everywhere...

  virtual int totaldocLength (int docID) const;

  int docLengthCounted(int docID);




  DocInfoList* docInfoList(int termID);

  TermInfoList* termInfoList(int docID);
  TermInfoList* termInfoListSeq(int docID);


  void setMesgStream(ostream * lemStream);
  void addKnownTerm( int termID, int position );
  int addUnknownTerm( InvFPTerm* term );
  int addUncachedTerm( InvFPTerm* term );

protected:
  void openDBs();
  void openSegments();
  void createDBs();

  void fullToc();
  bool docMgrIDs();
  record fetchDocumentRecord( int key ) const;
  void addDocumentLookup( int documentKey, const char* documentName );
  void addTermLookup( int termKey, const char* termSpelling );
  void addGeneralLookup( Keyfile& numberNameIndex, Keyfile& nameNumberIndex, 
                         int number, const char* name );
  InvFPDocList* internalDocInfoList(int termID);
  void _updateTermlist( InvFPDocList* curlist, int position );
  int _cacheSize();
  void _computeMemoryBounds( int memorySize );
  void _resetEstimatePoint();
  int* counts;    
  std::vector<std::string> names;
  float aveDocLen; 
  vector<std::string> docmgrs;
  ostream* msgstream;

  // All database handles are marked mutable since they sometimes
  // must be used to fetch values during const methods
  mutable Keyfile invlookup;
  
  // int <-> string mappings for documents and terms
  mutable Keyfile dIDs;
  mutable Keyfile dSTRs;
  mutable Keyfile tIDs;
  mutable Keyfile tSTRs;
  mutable File dtlookup; 
  ReadBuffer* dtlookupReadBuffer; 
  File writetlist; 

  char termKey[MAX_TERM_LENGTH];
  char docKey[MAX_DOCID_LENGTH];
  int _listsSize;
  int _memorySize;
  std::string name;
  vector<InvFPDocList*> invertlists; 
  vector<LocatedTerm> termlist; 
  int curdocmgr; 
  vector<DocumentManager*> docMgrs; 
  TermCache _cache;

  std::vector<File*> _segments;
  int _largestFlushedTermID;
  int _estimatePoint; 
};


#endif //_LEMUR_KEYFILE_INCINDEX_HPP

---