---

ngram.h

Go to the documentation of this file.

/*=====================================================================
                =======   COPYRIGHT NOTICE   =======
Copyright (C) 1996, Carnegie Mellon University, Cambridge University,
Ronald Rosenfeld and Philip Clarkson.

All rights reserved.

This software is made available for research purposes only.  It may be
redistributed freely for this purpose, in full or in part, provided
that this entire copyright notice is included on any copies of this
software and applications and derivations thereof.

This software is provided on an "as is" basis, without warranty of any
kind, either expressed or implied, as to any matter including, but not
limited to warranty of fitness of purpose, or merchantability, or
results obtained from use of this software.
======================================================================*/


/* Type and function definitions for general n_gram models */

#ifndef _NGRAM_H_
#define _NGRAM_H_

#include "rr_libs/sih.h"
#include "toolkit.h"

#define DEFAULT_COUNT_TABLE_SIZE 65535
#define DEFAULT_OOV_FRACTION 0.5
#define DEFAULT_DISC_RANGE_1 1
#define DEFAULT_DISC_RANGE_REST 7
#define DEFAULT_MIN_ALPHA -3.2
#define DEFAULT_MAX_ALPHA 2.5
#define DEFAULT_OUT_OF_RANGE_ALPHAS 10000

#define GOOD_TURING 1
#define ABSOLUTE 2
#define LINEAR 3
#define WITTEN_BELL 4

#define SPECIFIED 1
#define BUFFER 2
#define TWO_PASSES 3

#define KEY 65000

#define CLOSED_VOCAB 0
#define OPEN_VOCAB_1 1
#define OPEN_VOCAB_2 2

typedef unsigned short id__t; /* Double underscore, since id_t is
                                 already defined on some platforms */
typedef int count_t;   /* The count as read in, rather than its index 
                          in the count table. */
typedef unsigned short count_ind_t; /* The count's index in the count 
                                       table. */
typedef unsigned short bo_weight_t;
typedef unsigned short cutoff_t;
typedef int table_size_t;
typedef unsigned short index__t;
typedef double disc_val_t;
typedef double uni_probs_t;
typedef int ptr_tab_t;
typedef float four_byte_t;

typedef struct {
  unsigned short n; 
  id__t          *id_array; 
  count_t        count;
} ngram;

typedef struct {
  unsigned short count_table_size;
  int            *counts_array;
} count_table_t;


typedef struct {

  /* Language model type */

  unsigned short n;                
  int            version;

  /* Vocabulary stuff */

  sih_t          *vocab_ht;      
  unsigned short vocab_size;     
  char           **vocab;        
  unsigned short no_of_ccs;      
  /* Tree */

  table_size_t   *table_sizes;   
  id__t          **word_id;      
  count_ind_t    **count;        
  count_ind_t    *marg_counts;   
  int            **count4;       
  int            *marg_counts4;  
  bo_weight_t    **bo_weight;    
  four_byte_t    **bo_weight4;   
  index__t       **ind;          
  /* Two-byte alpha stuff */

  double         min_alpha;      
  double         max_alpha;      
  unsigned short out_of_range_alphas;  
  double         *alpha_array;
  unsigned short size_of_alpha_array;

  /* Count table */

  count_ind_t    count_table_size; 
  count_t        **count_table;    
  /* Index lookup tables */

  ptr_tab_t      **ptr_table;     
  unsigned short *ptr_table_size; 
  /* Discounting and cutoffs - note: some of these may not used,
     depending on the discounting techinque used. */

  unsigned short discounting_method;     
  cutoff_t       *cutoffs;               
  int            **freq_of_freq;         
  unsigned short *fof_size;              
  unsigned short *disc_range;            
  disc_val_t     **gt_disc_ratio;        
  disc_val_t     *lin_disc_ratio;        
  double         *abs_disc_const;        
  /* Unigram statistics */

  uni_probs_t    *uni_probs;             
  uni_probs_t    *uni_log_probs;         
  flag           *context_cue;           
  int            n_unigrams;             
  int            min_unicount;           
  /* Input files */

  char           *id_gram_filename;  
  FILE           *id_gram_fp;        
  char           *vocab_filename;    
  char           *context_cues_filename; 
  FILE           *context_cues_fp;       
  /* Output files */

  flag           write_arpa;      
  char           *arpa_filename;  
  FILE           *arpa_fp;        
  flag           write_bin;       
  char           *bin_filename;   
  FILE           *bin_fp;         
  /* Misc */

  int            *num_kgrams;     
  unsigned short vocab_type;      
  unsigned short first_id;        
  /* Once the tree has been constructed, the tables are indexed from 0
     to (num_kgrams[i]-1). */

  /* 1-gram tables are indexed from 0 to ng.vocab_size. */

  double         zeroton_fraction; 
  double         oov_fraction;
  flag           four_byte_alphas;
  flag           four_byte_counts;

} ng_t;

#endif

---