cd('e:\enron');n=0;
[md5s,paths]=textread('md5-filepath.txt','%s %s');
longFilePaths=paths;
for i=1:length(md5s),
    if i>1 & strcmp(md5s{i},md5s{i-1}), continue; end
    D=dir(['maildir' '\' paths{i}]);
    if D.bytes < 500, continue; end
    n=n+1;
    longFilePaths(n)=paths(i);
end
longFilePaths(n+1:end)=[];
clear md5s paths;
save longFilePaths longFilePaths
% remove header (except for subject),non-alpha chars, lowercase and save
load longFilePaths;
fidW=fopen(['maildirp\processedData'],'w');
for i=1:length(longFilePaths),
    fidR=fopen(['maildir'  '\' longFilePaths{i}    ],'r');
    sc=cell(0);bb=[];
    while 1
        tline = lower(fgetl(fidR));
        if ~ischar(tline), break, end
        if length(tline)>7 & strcmp(tline(1:8),'subject:'),
            tline(1:8)=[];
        elseif ~isempty(findstr(tline,':')), 
            continue; 
        end
        ind=find(tline=='.' | tline==',');
        tline(ind)=[];
        sc=[sc;str2wordCell(tline)];
    end
    scBits=isstrprop(sc,'alpha');
    for j=1:length(sc), bb=[bb all(scBits{j})];end
    sc(~bb)=[];
    fprintf(fidW,'%s\n',[num2str(i) cellStr2str(sc,' ')]);
    fclose(fidR);
    save progress i -ascii;
end
fclose(fidW);
% copy a subset of the lines
fidR=fopen(['maildirp\processedData'],'r');
fidW=fopen(['maildirp\processedDataShort'],'w');
for i=1:25000,  fprintf(fidW,'%s',fgets(fidR));end
fclose('all');
% Porter-stem
! porterStemmer maildirp\processedDataShort > maildirp\processedDataShortStemmed
% run the executable indexCorpus to create an index file
! indexCorpus maildirp\processedDataShort maildirp\indexedDataShort maildirp\vocab
! indexCorpus maildirp\processedDataShortStemmed maildirp\indexedDataShortStemmed maildirp\vocabStemmed
% save the sparse term occurance matrix to file
indexFile2termCountMatrix('maildirp\indexedDataShort',...
    'maildirp\matrixFile','maildirp\vocab',5,'stopWordList');
indexFile2termCountMatrix('maildirp\indexedDataShortStemmed',...
    'maildirp\matrixFileStemmed','maildirp\vocabStemmed',5,'stopWordList');

