package ciranda;

import java.io.File;
import java.io.Serializable;
import java.util.Iterator;

import ciranda.features.CommitFE;
import ciranda.features.DdataFE;
import ciranda.features.DeliverFE;
import ciranda.features.FeatureExtractor;
import ciranda.features.MeetFE;
import ciranda.features.ProposeFE;
import ciranda.features.RequestFE;
import edu.cmu.minorthird.classify.*;
import edu.cmu.minorthird.classify.experiments.Evaluation;
import edu.cmu.minorthird.classify.experiments.Expt;
import edu.cmu.minorthird.classify.experiments.Tester;
import edu.cmu.minorthird.classify.transform.*;
import edu.cmu.minorthird.text.*;
import edu.cmu.minorthird.text.gui.TextBaseEditor;
import edu.cmu.minorthird.text.learn.*;
import edu.cmu.minorthird.ui.Recommended.MaxEntLearner;
import edu.cmu.minorthird.ui.Recommended.NaiveBayes;
import edu.cmu.minorthird.ui.Recommended.SVMLearner;
import edu.cmu.minorthird.util.*;
import edu.cmu.minorthird.util.gui.ViewerFrame;

/**
 * @author vitor carvalho
 * March 2005
 * Trains new speech act classifiers - 
 * To use, you need to provide data and labels according to minorthird format
 * 
 * The input messages must contain body of email between <body> </body> tags and, optionally, 
 * the subject of the email between <subject> </subject> tags. 
 * Remember: remove the forwarded part of messages. Remove the reply-to lines(quotes) from the body
 *
 */
public class SpeechActTrainerbyLine {

	private String outputFile;
	private String act;
	private boolean hasXMLTags = true;

	//constructors
	public SpeechActTrainerbyLine(String act, String outfile, boolean hasXml){
		outputFile = outfile; this.act = act; hasXMLTags = hasXml;
	}
	
	public void train(){
        String modelName = act+"_"+outputFile;
		Dataset dataset = new BasicDataset();
		Dataset testdataset = new BasicDataset();
		TextLabels labels; TextLabels testlabels;
		try {
		  labels = this.myreadBsh(new File("example1.base"));
//		  testlabels = LineProcessingUtil.readBsh(new File("C:/m3test/collective/n04f4/data"), new File("C:/m3test/collective/n04f4/env/newalllabelsn04f4.env"));
//		  testlabels = LineProcessingUtil.readBsh(new File("C:/m3test/collective/n02f2/data"), new File("C:/m3test/collective/n02f2/env/newalllabelsn02f2.env"));
//		  labels = LineProcessingUtil.readBsh(new File("C:/m3test/collective/n03f2/data"), new File("C:/m3test/collective/n03f2/env/newalllabelsn03f2.env"));
//          labels = LineProcessingUtil.readBsh(new File(datarepository), new File(envfile+"my"+act+".env"));
//		  testlabels = LineProcessingUtil.readBsh(new File(testdatarepository), new File(testenvfile+"newalllabelsn02f2.env"));
//		  testlabels = CirandaUtils.readBsh()
		  TextBase base = labels.getTextBase();
		  TextBaseEditor.edit((MutableTextLabels)labels,new File("foo"));
//		  System.exit(0);

//
		  //iterate over documents
  		  for(Iterator it = base.documentSpanIterator(); it.hasNext();){
                        Span bodyspan = (Span)it.next();
		  	String docid = bodyspan.getDocumentId();
                        Span docspan = labels.getTextBase().documentSpan(docid);
		  	FeatureExtractor myfe =null;
		  	if(act.startsWith("meet")) myfe = new MeetFE(bodyspan.asString());
		  	else if(act.startsWith("Cmt")) myfe = new CommitFE(bodyspan.asString());
		  	else if(act.startsWith("Req"))myfe = new RequestFE(bodyspan.asString());//body only
		  	else if(act.startsWith("Prop"))myfe = new ProposeFE(bodyspan.asString());//body only
		  	else if(act.startsWith("Dlv"))myfe = new DeliverFE(bodyspan.asString());//body only
		  	else if(act.startsWith("dData"))myfe = new DdataFE(bodyspan.asString());
		  	else if(act.startsWith("Cry"))myfe = new DdataFE(bodyspan.asString());
		  	else {
		  		System.out.println("ERROR: Unknown act "+act);
		  	//	System.exit(0);
		  	}
//		  	FeatureExtractor myfe = new RequestFE(bodyspan.asString(),subject);
		  	Instance ins = myfe.string2Instance();

		  	//extract label
		  	ClassLabel mylabel = new ClassLabel();
		  	mylabel = labels.hasType(docspan,act) ? ClassLabel.binaryLabel(+1) : ClassLabel.binaryLabel(-1);

		  	//add to dataset
		  	dataset.add(new Example(ins,mylabel));
		  }
		  System.out.println("Dataset size = "+dataset.size());
//		  TextBaseEditor.edit((MutableTextLabels)labels,new File("foo"));
//		  System.out.println("DATA = "+dataset.toString());

		  //only consider features with coutn superior to 1
		  FrequencyBasedTransformLearner filt = new FrequencyBasedTransformLearner(1,"document");
		  AbstractInstanceTransform ait = (AbstractInstanceTransform)filt.batchTrain(dataset);
		  dataset = ait.transform(dataset);
//
////		  info gain seems more efficient than chi-square in low count features
		  InfoGainTransformLearner igt = new InfoGainTransformLearner();
		  InfoGainInstanceTransform igi = (InfoGainInstanceTransform)igt.batchTrain(dataset);
//		  int cutoff = dataset.size()*1;
		  int cutoff = 10000;
		  igi.setNumberOfFeatures(cutoff);
		  dataset = igi.transform(dataset);
//
//		  ChiSquareTransformLearner igt = new ChiSquareTransformLearner();
//		  ChiSquareInstanceTransform filter = (ChiSquareInstanceTransform)igt.batchTrain(dataset);
//		  int cutoff = dataset.size();
////		  int cutoff = 300;
//		  filter.setNumberOfFeatures(cutoff);
////		  System.out.println(igi.);
//		  dataset = filter.transform(dataset);
//		  System.out.println(filter.toString(200));

//		  System.exit(0);


//OVER TEST LABELS
//		  TextBase testbase = testlabels.getTextBase();
////		  iterate over documents
//		  for(Iterator it = testlabels.instanceIterator("body"); it.hasNext();){
//		  	Span bodyspan = (Span)it.next();
//		  	String docid = bodyspan.getDocumentId();
//
//		  	//extract features
//		  	FeatureExtractor myfe = new RequestFE(bodyspan.asString());
//		  	Instance ins = myfe.string2Instance();
//
//		  	//extract label
//		  	Span docspan = testlabels.getTextBase().documentSpan(docid);
//		  	ClassLabel mylabel = new ClassLabel();
//		  	mylabel = testlabels.hasType(docspan,act) ? ClassLabel.binaryLabel(+1) : ClassLabel.binaryLabel(-1);
//
//		  	//add to dataset
//		  	testdataset.add(new Example(ins,mylabel));
//		  }
//		  System.out.println("TestDataset size = "+testdataset.size());

//		  System.exit(0);

		  ClassifierLearner learner = new MaxEntLearner();
//		  ClassifierLearner learner = new SVMLearner();
//		  ClassifierLearner learner = new NaiveBayes();

          //if you want to do crosssvalidation tests
//		  Splitter split = Expt.toSplitter("k5");
//		  Evaluation eval = Tester.evaluate(learner, dataset, split);
//		  Evaluation eval = Tester.evaluate(learner, dataset, testdataset);
//		  ViewerFrame frame = new ViewerFrame("numeric demo", eval.toGUI());
//		  eval.summarize();

		  System.out.println("training the Model...");
		  Classifier cl = new DatasetClassifierTeacher(dataset).train(learner);
		  System.out.println("saving model in file..." + modelName);
		  IOUtil.saveSerialized((Serializable) cl, new File(modelName));
	    }
		catch (Exception e) {
			e.printStackTrace();
		}

	}

	//don't use this
	public TextLabels myreadBsh(File dir) throws Exception {
		System.out.println("reading input file..."+dir);
		TextBaseLoader tbl = new TextBaseLoader(TextBaseLoader.DOC_PER_LINE, true);
		tbl.load(dir);
		TextLabels lala  = tbl.getLabels();
		return lala;
	}

	public static void main(String[] args) {
		String act = args[0];
		String filename = args[1];
		SpeechActTrainerbyLine sat = new SpeechActTrainerbyLine(act,filename,false);
		sat.train();
	}
}
